Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
@@ -16,6 +16,7 @@ from vllm.model_executor.determinism.batch_invariant_configs import (
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.mem_utils import get_max_shared_memory_bytes
from vllm.utils.nccl import pin_nccl_env
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import is_torch_equal_or_newer
@@ -1139,25 +1140,33 @@ def enable_batch_invariant_mode():
def override_envs_for_invariance():
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
# Only the 1-stage kernel has a size- and rank-independent reduction order.
os.environ["VLLM_CUSTOM_ALLREDUCE_ALGO"] = "1stage"
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
# NCCL determinism settings
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
# unlike the pins below, it also applies to communicators shared with other
# processes.
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
os.environ["NCCL_COLLNET_ENABLE"] = "0"
os.environ["NCCL_NVLS_ENABLE"] = "0"
os.environ["NCCL_P2P_NET_DISABLE"] = "1"
os.environ["NCCL_MIN_NCHANNELS"] = "1"
os.environ["NCCL_MAX_NCHANNELS"] = "1"
os.environ["NCCL_PROTO"] = "Simple"
# NCCL >= 2.31 zero-fills the algorithm table of every collective when
# NCCL_ALGO is set and re-enables only the named ones; together with the
# NCCL_PROTO above, collectives not named here end up with no algorithm
# and fail with ncclInvalidUsage. Re-enable Ring and Tree for all
# collectives, then pin AllReduce to Tree for determinism.
os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree"
os.environ["NCCL_NTHREADS"] = "1"
os.environ["NCCL_SOCKET_NTHREADS"] = "1"
pin_nccl_env(
{
"NCCL_COLLNET_ENABLE": "0",
"NCCL_NVLS_ENABLE": "0",
"NCCL_P2P_NET_DISABLE": "1",
"NCCL_MIN_NCHANNELS": "1",
"NCCL_MAX_NCHANNELS": "1",
"NCCL_PROTO": "Simple",
# NCCL >= 2.31 zero-fills the algorithm table of every collective
# when NCCL_ALGO is set and re-enables only the named ones; together
# with the NCCL_PROTO above, collectives not named here end up with
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
# Tree for all collectives, then pin AllReduce to Tree.
"NCCL_ALGO": "ring,tree;allreduce:tree",
"NCCL_NTHREADS": "1",
"NCCL_SOCKET_NTHREADS": "1",
}
)
# torch.compile settings
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
@@ -480,6 +480,621 @@ _BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: dict[
(2048, _MatmulMConfig(64, 128, 4, 3)),
),
),
(6144, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp2_gate_up_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 3)),
(4, _MatmulMConfig(16, 128, 8, 5)),
(8, _MatmulMConfig(16, 128, 4, 5)),
(16, _MatmulMConfig(16, 32, 4, 5)),
(32, _MatmulMConfig(32, 128, 8, 5)),
(64, _MatmulMConfig(32, 128, 4, 4)),
(256, _MatmulMConfig(128, 128, 4, 3)),
(512, _MatmulMConfig(64, 64, 4, 5)),
(1024, _MatmulMConfig(128, 64, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(2048, 3072): _MatmulShapeConfig( # Qwen3-1.7B tp2_down_proj
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 5)),
(4, _MatmulMConfig(16, 64, 4, 3)),
(8, _MatmulMConfig(16, 64, 8, 4)),
(16, _MatmulMConfig(16, 64, 8, 3)),
(32, _MatmulMConfig(16, 64, 8, 3)),
(64, _MatmulMConfig(16, 64, 4, 5)),
(256, _MatmulMConfig(64, 64, 8, 3)),
(512, _MatmulMConfig(64, 128, 4, 3)),
(1024, _MatmulMConfig(64, 64, 4, 3)),
(2048, _MatmulMConfig(64, 128, 4, 3)),
),
),
(2048, 1024): _MatmulShapeConfig( # Qwen3-1.7B tp2_o_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 256, 8, 2)),
(4, _MatmulMConfig(16, 32, 8, 4)),
(8, _MatmulMConfig(32, 128, 8, 5)),
(16, _MatmulMConfig(32, 128, 4, 4)),
(32, _MatmulMConfig(64, 128, 4, 3)),
(64, _MatmulMConfig(16, 256, 4, 3)),
(256, _MatmulMConfig(32, 128, 4, 3)),
(512, _MatmulMConfig(64, 128, 4, 4)),
(1024, _MatmulMConfig(128, 128, 8, 3)),
(2048, _MatmulMConfig(64, 128, 4, 5)),
),
),
(75968, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp2_lm_head
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 256, 8, 3)),
(4, _MatmulMConfig(16, 256, 8, 3)),
(8, _MatmulMConfig(16, 256, 8, 3)),
(16, _MatmulMConfig(16, 256, 4, 3)),
(32, _MatmulMConfig(32, 256, 8, 3)),
(64, _MatmulMConfig(64, 256, 8, 3)),
(256, _MatmulMConfig(64, 128, 4, 5)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(3072, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp4_gate_up_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 256, 4, 3)),
(4, _MatmulMConfig(32, 128, 4, 5)),
(8, _MatmulMConfig(16, 128, 8, 3)),
(16, _MatmulMConfig(32, 128, 8, 4)),
(32, _MatmulMConfig(32, 128, 8, 5)),
(64, _MatmulMConfig(64, 32, 8, 3)),
(256, _MatmulMConfig(128, 64, 8, 3)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(64, 64, 4, 5)),
(2048, _MatmulMConfig(64, 128, 8, 4)),
),
),
(2048, 1536): _MatmulShapeConfig( # Qwen3-1.7B tp4_down_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 2)),
(4, _MatmulMConfig(16, 256, 4, 3)),
(8, _MatmulMConfig(32, 128, 4, 2)),
(16, _MatmulMConfig(32, 32, 8, 5)),
(32, _MatmulMConfig(32, 32, 8, 4)),
(64, _MatmulMConfig(32, 128, 4, 4)),
(256, _MatmulMConfig(64, 64, 4, 3)),
(512, _MatmulMConfig(128, 64, 8, 4)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 64, 8, 4)),
),
),
(1024, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp4_qkv_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(32, 128, 8, 4)),
(4, _MatmulMConfig(16, 32, 8, 4)),
(8, _MatmulMConfig(16, 32, 4, 4)),
(16, _MatmulMConfig(32, 128, 8, 3)),
(32, _MatmulMConfig(32, 128, 8, 4)),
(64, _MatmulMConfig(32, 64, 4, 5)),
(256, _MatmulMConfig(64, 32, 4, 3)),
(512, _MatmulMConfig(64, 64, 4, 3)),
(1024, _MatmulMConfig(64, 128, 4, 5)),
(2048, _MatmulMConfig(128, 128, 8, 3)),
),
),
(2048, 512): _MatmulShapeConfig( # Qwen3-1.7B tp4_o_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(32, 32, 4, 5)),
(4, _MatmulMConfig(64, 128, 4, 2)),
(8, _MatmulMConfig(16, 128, 4, 2)),
(16, _MatmulMConfig(16, 256, 8, 3)),
(32, _MatmulMConfig(64, 32, 8, 4)),
(64, _MatmulMConfig(16, 128, 8, 4)),
(256, _MatmulMConfig(64, 64, 8, 5)),
(512, _MatmulMConfig(32, 128, 8, 4)),
(1024, _MatmulMConfig(64, 64, 8, 3)),
(2048, _MatmulMConfig(64, 128, 8, 3)),
),
),
(37984, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp4_lm_head
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 4)),
(4, _MatmulMConfig(16, 128, 8, 5)),
(8, _MatmulMConfig(16, 128, 4, 5)),
(16, _MatmulMConfig(16, 128, 8, 4)),
(32, _MatmulMConfig(32, 128, 4, 5)),
(64, _MatmulMConfig(64, 128, 4, 5)),
(256, _MatmulMConfig(64, 128, 4, 5)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(1536, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp8_gate_up_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(32, 128, 8, 3)),
(4, _MatmulMConfig(16, 256, 4, 3)),
(8, _MatmulMConfig(16, 32, 8, 5)),
(16, _MatmulMConfig(32, 32, 8, 3)),
(32, _MatmulMConfig(32, 128, 8, 4)),
(64, _MatmulMConfig(16, 128, 8, 4)),
(256, _MatmulMConfig(32, 128, 8, 3)),
(512, _MatmulMConfig(64, 128, 8, 4)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(64, 64, 4, 4)),
),
),
(2048, 768): _MatmulShapeConfig( # Qwen3-1.7B tp8_down_proj
block_k=128,
m_buckets=(
(1, _MatmulMConfig(32, 64, 4, 3)),
(4, _MatmulMConfig(32, 64, 8, 3)),
(8, _MatmulMConfig(16, 32, 4, 5)),
(16, _MatmulMConfig(32, 64, 4, 2)),
(32, _MatmulMConfig(16, 64, 8, 3)),
(64, _MatmulMConfig(16, 32, 8, 2)),
(256, _MatmulMConfig(32, 32, 4, 4)),
(512, _MatmulMConfig(32, 64, 4, 5)),
(1024, _MatmulMConfig(64, 128, 4, 3)),
(2048, _MatmulMConfig(64, 128, 4, 3)),
),
),
(512, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp8_qkv_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(64, 64, 8, 5)),
(4, _MatmulMConfig(16, 32, 4, 4)),
(8, _MatmulMConfig(16, 32, 4, 5)),
(16, _MatmulMConfig(32, 128, 8, 3)),
(32, _MatmulMConfig(64, 64, 8, 5)),
(64, _MatmulMConfig(32, 128, 8, 4)),
(256, _MatmulMConfig(16, 128, 4, 4)),
(512, _MatmulMConfig(32, 64, 8, 3)),
(1024, _MatmulMConfig(64, 64, 4, 4)),
(2048, _MatmulMConfig(64, 128, 4, 4)),
),
),
(2048, 256): _MatmulShapeConfig( # Qwen3-1.7B tp8_o_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(32, 128, 8, 2)),
(4, _MatmulMConfig(16, 128, 4, 2)),
(8, _MatmulMConfig(32, 256, 4, 2)),
(16, _MatmulMConfig(32, 32, 8, 4)),
(32, _MatmulMConfig(16, 256, 4, 3)),
(64, _MatmulMConfig(128, 32, 8, 3)),
(256, _MatmulMConfig(16, 32, 8, 2)),
(512, _MatmulMConfig(16, 32, 4, 3)),
(1024, _MatmulMConfig(64, 256, 4, 3)),
(2048, _MatmulMConfig(64, 128, 8, 4)),
),
),
(18992, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp8_lm_head
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 64, 8, 5)),
(4, _MatmulMConfig(16, 128, 8, 5)),
(8, _MatmulMConfig(16, 128, 8, 3)),
(16, _MatmulMConfig(16, 128, 4, 5)),
(32, _MatmulMConfig(32, 128, 8, 5)),
(64, _MatmulMConfig(64, 128, 8, 4)),
(256, _MatmulMConfig(128, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 8, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(9728, 2560): _MatmulShapeConfig( # Qwen3-4B tp2_gate_up_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 4)),
(4, _MatmulMConfig(16, 64, 8, 3)),
(8, _MatmulMConfig(16, 128, 4, 5)),
(16, _MatmulMConfig(16, 32, 4, 5)),
(32, _MatmulMConfig(32, 64, 8, 4)),
(64, _MatmulMConfig(32, 128, 4, 3)),
(256, _MatmulMConfig(128, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(64, 128, 4, 5)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(2560, 4864): _MatmulShapeConfig( # Qwen3-4B tp2_down_proj
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 32, 4, 3)),
(4, _MatmulMConfig(16, 128, 8, 3)),
(8, _MatmulMConfig(16, 64, 8, 4)),
(16, _MatmulMConfig(32, 64, 8, 4)),
(32, _MatmulMConfig(16, 64, 4, 3)),
(64, _MatmulMConfig(32, 32, 8, 4)),
(256, _MatmulMConfig(64, 64, 4, 3)),
(512, _MatmulMConfig(64, 128, 4, 3)),
(1024, _MatmulMConfig(64, 128, 4, 3)),
(2048, _MatmulMConfig(64, 128, 4, 3)),
),
),
(3072, 2560): _MatmulShapeConfig( # Qwen3-4B tp2_qkv_proj
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 3)),
(4, _MatmulMConfig(16, 32, 8, 2)),
(8, _MatmulMConfig(16, 32, 8, 4)),
(16, _MatmulMConfig(16, 64, 4, 3)),
(32, _MatmulMConfig(32, 64, 4, 4)),
(64, _MatmulMConfig(16, 128, 4, 3)),
(256, _MatmulMConfig(64, 128, 4, 3)),
(512, _MatmulMConfig(64, 64, 4, 3)),
(1024, _MatmulMConfig(64, 128, 4, 3)),
(2048, _MatmulMConfig(64, 128, 4, 3)),
),
),
(2560, 2048): _MatmulShapeConfig( # Qwen3-4B tp2_o_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(32, 128, 8, 4)),
(4, _MatmulMConfig(32, 128, 4, 4)),
(8, _MatmulMConfig(16, 128, 4, 3)),
(16, _MatmulMConfig(32, 128, 4, 5)),
(32, _MatmulMConfig(16, 128, 8, 4)),
(64, _MatmulMConfig(16, 128, 8, 4)),
(256, _MatmulMConfig(128, 32, 8, 5)),
(512, _MatmulMConfig(128, 64, 8, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 8, 3)),
),
),
(75968, 2560): _MatmulShapeConfig( # Qwen3-4B tp2_lm_head
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 64, 8, 5)),
(4, _MatmulMConfig(16, 64, 8, 5)),
(8, _MatmulMConfig(16, 64, 4, 5)),
(16, _MatmulMConfig(16, 64, 4, 5)),
(32, _MatmulMConfig(32, 64, 4, 5)),
(64, _MatmulMConfig(64, 256, 4, 3)),
(256, _MatmulMConfig(64, 128, 4, 5)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 8, 3)),
),
),
(4864, 2560): _MatmulShapeConfig( # Qwen3-4B tp4_gate_up_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 5)),
(4, _MatmulMConfig(16, 128, 4, 4)),
(8, _MatmulMConfig(16, 32, 8, 4)),
(16, _MatmulMConfig(16, 128, 8, 4)),
(32, _MatmulMConfig(16, 128, 4, 5)),
(64, _MatmulMConfig(16, 128, 4, 3)),
(256, _MatmulMConfig(64, 128, 4, 5)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(2560, 2432): _MatmulShapeConfig( # Qwen3-4B tp4_down_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 4)),
(4, _MatmulMConfig(16, 32, 4, 5)),
(8, _MatmulMConfig(32, 64, 8, 4)),
(16, _MatmulMConfig(32, 128, 8, 5)),
(32, _MatmulMConfig(16, 128, 4, 4)),
(64, _MatmulMConfig(64, 32, 4, 3)),
(256, _MatmulMConfig(32, 128, 8, 4)),
(512, _MatmulMConfig(64, 128, 4, 5)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(1536, 2560): _MatmulShapeConfig( # Qwen3-4B tp4_qkv_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 4)),
(4, _MatmulMConfig(16, 32, 8, 3)),
(8, _MatmulMConfig(16, 32, 4, 4)),
(16, _MatmulMConfig(32, 128, 8, 3)),
(32, _MatmulMConfig(16, 128, 4, 3)),
(64, _MatmulMConfig(16, 64, 8, 4)),
(256, _MatmulMConfig(128, 32, 4, 5)),
(512, _MatmulMConfig(64, 128, 4, 5)),
(1024, _MatmulMConfig(64, 256, 4, 3)),
(2048, _MatmulMConfig(64, 128, 4, 4)),
),
),
(2560, 1024): _MatmulShapeConfig( # Qwen3-4B tp4_o_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(32, 128, 8, 5)),
(4, _MatmulMConfig(16, 32, 8, 3)),
(8, _MatmulMConfig(16, 128, 4, 3)),
(16, _MatmulMConfig(32, 256, 4, 3)),
(32, _MatmulMConfig(16, 128, 4, 2)),
(64, _MatmulMConfig(32, 256, 8, 3)),
(256, _MatmulMConfig(32, 128, 4, 5)),
(512, _MatmulMConfig(64, 128, 8, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(37984, 2560): _MatmulShapeConfig( # Qwen3-4B tp4_lm_head
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 3)),
(4, _MatmulMConfig(16, 128, 8, 3)),
(8, _MatmulMConfig(16, 128, 4, 3)),
(16, _MatmulMConfig(16, 32, 4, 5)),
(32, _MatmulMConfig(16, 32, 4, 5)),
(64, _MatmulMConfig(32, 32, 4, 5)),
(256, _MatmulMConfig(64, 128, 4, 3)),
(512, _MatmulMConfig(64, 128, 4, 3)),
(1024, _MatmulMConfig(64, 128, 4, 3)),
(2048, _MatmulMConfig(64, 128, 4, 3)),
),
),
(2432, 2560): _MatmulShapeConfig( # Qwen3-4B tp8_gate_up_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 4, 5)),
(4, _MatmulMConfig(32, 128, 4, 3)),
(8, _MatmulMConfig(16, 128, 8, 3)),
(16, _MatmulMConfig(16, 128, 4, 4)),
(32, _MatmulMConfig(16, 128, 8, 4)),
(64, _MatmulMConfig(16, 128, 4, 3)),
(256, _MatmulMConfig(64, 64, 4, 5)),
(512, _MatmulMConfig(64, 128, 4, 4)),
(1024, _MatmulMConfig(128, 128, 8, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(2560, 1216): _MatmulShapeConfig( # Qwen3-4B tp8_down_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 256, 8, 2)),
(4, _MatmulMConfig(16, 128, 8, 4)),
(8, _MatmulMConfig(16, 32, 4, 3)),
(16, _MatmulMConfig(32, 32, 4, 3)),
(32, _MatmulMConfig(16, 128, 8, 4)),
(64, _MatmulMConfig(32, 128, 8, 4)),
(256, _MatmulMConfig(32, 128, 4, 5)),
(512, _MatmulMConfig(64, 128, 8, 4)),
(1024, _MatmulMConfig(64, 256, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(768, 2560): _MatmulShapeConfig( # Qwen3-4B tp8_qkv_proj
block_k=128,
m_buckets=(
(1, _MatmulMConfig(32, 32, 4, 2)),
(4, _MatmulMConfig(16, 64, 8, 3)),
(8, _MatmulMConfig(16, 64, 8, 4)),
(16, _MatmulMConfig(16, 64, 4, 5)),
(32, _MatmulMConfig(32, 64, 8, 4)),
(64, _MatmulMConfig(16, 64, 8, 4)),
(256, _MatmulMConfig(64, 32, 4, 5)),
(512, _MatmulMConfig(64, 64, 8, 3)),
(1024, _MatmulMConfig(64, 128, 4, 3)),
(2048, _MatmulMConfig(64, 64, 4, 3)),
),
),
(2560, 512): _MatmulShapeConfig( # Qwen3-4B tp8_o_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 4)),
(4, _MatmulMConfig(16, 32, 4, 2)),
(8, _MatmulMConfig(16, 128, 4, 5)),
(16, _MatmulMConfig(32, 32, 4, 5)),
(32, _MatmulMConfig(32, 256, 4, 3)),
(64, _MatmulMConfig(32, 128, 8, 5)),
(256, _MatmulMConfig(16, 256, 4, 2)),
(512, _MatmulMConfig(32, 128, 8, 4)),
(1024, _MatmulMConfig(128, 128, 8, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(18992, 2560): _MatmulShapeConfig( # Qwen3-4B tp8_lm_head
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 64, 8, 4)),
(4, _MatmulMConfig(16, 64, 8, 5)),
(8, _MatmulMConfig(16, 128, 8, 3)),
(16, _MatmulMConfig(16, 128, 8, 5)),
(32, _MatmulMConfig(32, 128, 8, 5)),
(64, _MatmulMConfig(64, 128, 8, 4)),
(256, _MatmulMConfig(128, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(12288, 4096): _MatmulShapeConfig( # Qwen3-8B tp2_gate_up_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 3)),
(4, _MatmulMConfig(16, 128, 8, 5)),
(8, _MatmulMConfig(16, 128, 8, 4)),
(16, _MatmulMConfig(16, 128, 4, 4)),
(32, _MatmulMConfig(32, 128, 4, 4)),
(64, _MatmulMConfig(64, 128, 4, 5)),
(256, _MatmulMConfig(64, 64, 4, 5)),
(512, _MatmulMConfig(128, 64, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(4096, 6144): _MatmulShapeConfig( # Qwen3-8B tp2_down_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 5)),
(4, _MatmulMConfig(16, 32, 8, 5)),
(8, _MatmulMConfig(16, 32, 8, 3)),
(16, _MatmulMConfig(16, 32, 4, 5)),
(32, _MatmulMConfig(16, 64, 8, 4)),
(64, _MatmulMConfig(32, 64, 4, 5)),
(256, _MatmulMConfig(64, 128, 4, 5)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 64, 8, 4)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(3072, 4096): _MatmulShapeConfig( # Qwen3-8B tp2_qkv_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 4)),
(4, _MatmulMConfig(16, 32, 8, 3)),
(8, _MatmulMConfig(16, 128, 4, 3)),
(16, _MatmulMConfig(16, 32, 4, 4)),
(32, _MatmulMConfig(32, 32, 8, 5)),
(64, _MatmulMConfig(16, 128, 8, 4)),
(256, _MatmulMConfig(64, 128, 4, 5)),
(512, _MatmulMConfig(128, 128, 8, 3)),
(1024, _MatmulMConfig(64, 64, 4, 5)),
(2048, _MatmulMConfig(128, 64, 4, 3)),
),
),
(75968, 4096): _MatmulShapeConfig( # Qwen3-8B tp2_lm_head
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 256, 8, 3)),
(4, _MatmulMConfig(16, 256, 8, 3)),
(8, _MatmulMConfig(16, 256, 8, 3)),
(16, _MatmulMConfig(16, 256, 8, 3)),
(32, _MatmulMConfig(32, 256, 8, 3)),
(64, _MatmulMConfig(64, 256, 8, 3)),
(256, _MatmulMConfig(128, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 8, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(4096, 3072): _MatmulShapeConfig( # Qwen3-8B tp4_down_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 256, 8, 3)),
(4, _MatmulMConfig(16, 128, 4, 5)),
(8, _MatmulMConfig(16, 128, 4, 4)),
(16, _MatmulMConfig(16, 64, 4, 5)),
(32, _MatmulMConfig(32, 128, 4, 3)),
(64, _MatmulMConfig(16, 128, 4, 4)),
(256, _MatmulMConfig(64, 128, 4, 5)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 64, 8, 4)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(1536, 4096): _MatmulShapeConfig( # Qwen3-8B tp4_qkv_proj
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 64, 4, 5)),
(4, _MatmulMConfig(16, 64, 8, 3)),
(8, _MatmulMConfig(32, 64, 4, 5)),
(16, _MatmulMConfig(16, 64, 8, 4)),
(32, _MatmulMConfig(16, 32, 8, 3)),
(64, _MatmulMConfig(16, 64, 8, 5)),
(256, _MatmulMConfig(64, 64, 4, 3)),
(512, _MatmulMConfig(64, 128, 4, 3)),
(1024, _MatmulMConfig(64, 64, 4, 3)),
(2048, _MatmulMConfig(64, 128, 4, 3)),
),
),
(4096, 1024): _MatmulShapeConfig( # Qwen3-8B tp4_o_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 4)),
(4, _MatmulMConfig(32, 128, 8, 4)),
(8, _MatmulMConfig(16, 256, 8, 3)),
(16, _MatmulMConfig(16, 128, 8, 3)),
(32, _MatmulMConfig(16, 128, 4, 4)),
(64, _MatmulMConfig(64, 64, 4, 3)),
(256, _MatmulMConfig(128, 64, 4, 3)),
(512, _MatmulMConfig(64, 64, 4, 5)),
(1024, _MatmulMConfig(128, 64, 4, 4)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(37984, 4096): _MatmulShapeConfig( # Qwen3-8B tp4_lm_head
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 3)),
(4, _MatmulMConfig(16, 128, 4, 5)),
(8, _MatmulMConfig(16, 128, 4, 5)),
(16, _MatmulMConfig(16, 128, 8, 3)),
(32, _MatmulMConfig(32, 128, 8, 3)),
(64, _MatmulMConfig(64, 128, 8, 3)),
(256, _MatmulMConfig(64, 128, 4, 5)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(4096, 1536): _MatmulShapeConfig( # Qwen3-8B tp8_down_proj
block_k=32,
m_buckets=(
(1, _MatmulMConfig(32, 128, 8, 5)),
(4, _MatmulMConfig(16, 256, 4, 5)),
(8, _MatmulMConfig(16, 64, 4, 4)),
(16, _MatmulMConfig(16, 128, 8, 3)),
(32, _MatmulMConfig(16, 64, 4, 3)),
(64, _MatmulMConfig(16, 128, 4, 4)),
(256, _MatmulMConfig(64, 128, 4, 4)),
(512, _MatmulMConfig(128, 128, 8, 5)),
(1024, _MatmulMConfig(128, 256, 8, 4)),
(2048, _MatmulMConfig(128, 128, 8, 4)),
),
),
(768, 4096): _MatmulShapeConfig( # Qwen3-8B tp8_qkv_proj
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 5)),
(4, _MatmulMConfig(16, 32, 8, 3)),
(8, _MatmulMConfig(32, 64, 4, 5)),
(16, _MatmulMConfig(32, 64, 8, 3)),
(32, _MatmulMConfig(32, 32, 8, 5)),
(64, _MatmulMConfig(64, 32, 8, 3)),
(256, _MatmulMConfig(32, 64, 4, 4)),
(512, _MatmulMConfig(64, 64, 8, 3)),
(1024, _MatmulMConfig(64, 128, 4, 3)),
(2048, _MatmulMConfig(64, 64, 4, 3)),
),
),
(4096, 512): _MatmulShapeConfig( # Qwen3-8B tp8_o_proj
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 2)),
(4, _MatmulMConfig(32, 128, 4, 2)),
(8, _MatmulMConfig(16, 256, 4, 2)),
(16, _MatmulMConfig(32, 256, 4, 3)),
(32, _MatmulMConfig(32, 256, 8, 3)),
(64, _MatmulMConfig(32, 128, 4, 5)),
(256, _MatmulMConfig(64, 32, 8, 4)),
(512, _MatmulMConfig(128, 128, 8, 3)),
(1024, _MatmulMConfig(128, 64, 4, 4)),
(2048, _MatmulMConfig(128, 128, 8, 3)),
),
),
(18992, 4096): _MatmulShapeConfig( # Qwen3-8B tp8_lm_head
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 64, 8, 5)),
(4, _MatmulMConfig(16, 64, 8, 4)),
(8, _MatmulMConfig(16, 64, 8, 5)),
(16, _MatmulMConfig(16, 128, 4, 3)),
(32, _MatmulMConfig(32, 64, 8, 5)),
(64, _MatmulMConfig(64, 128, 8, 3)),
(256, _MatmulMConfig(128, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
},
}
@@ -213,5 +213,10 @@ def _get_isa_hint(dtype: torch.dtype) -> str:
return "amx"
elif current_platform.get_cpu_architecture() == CpuArchEnum.RISCV:
return "rvv"
elif (
dtype == torch.bfloat16
and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC
):
return "vsx"
else:
return "vec"
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zentorch W4A16 GPTQ weight-only-quantized linear kernel for AMD Zen CPUs.
"""Zentorch int4 weight-quantized linear kernels for AMD Zen CPUs.
Selected by ``choose_mp_linear_kernel`` ahead of the generic oneDNN-backed
``CPUWNA16LinearKernel``. When ``can_implement`` rejects a layer, the selector
@@ -9,12 +9,13 @@ falls through to the next kernel in ``_POSSIBLE_KERNELS[PlatformEnum.CPU]``.
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from .cpu import CPUWNA16LinearKernel
from .cpu import _CPUWNA16_SUPPORTED_QUANT_TYPES, CPUWNA16LinearKernel
from .MPLinearKernel import MPLinearLayerConfig
logger = init_logger(__name__)
@@ -34,17 +35,35 @@ def _import_unpack_from_int32():
class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
"""W4A16 GPTQ kernel backed by ``torch.ops.zentorch.zentorch_woq_linear``."""
"""Int4 kernel backed by ``zentorch_dynamic_qlinear`` (DA8W4) or
``zentorch_woq_linear`` (W4A16)."""
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
ok, reason = super().can_implement(c)
if not ok:
return ok, reason
if not current_platform.is_zen_cpu():
return False, "ZentorchWNA16 requires an AMD Zen CPU."
# Not super(): its N/K multiple-of-32 rule is a oneDNN packing
# constraint the zentorch repack does not share.
if c.weight_type not in _CPUWNA16_SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type ({c.weight_type}) not supported by "
"CPUWNA16, supported types are: "
f"{_CPUWNA16_SUPPORTED_QUANT_TYPES}",
)
# -1 is per-channel, one group spanning K. Anything else has to divide
# K, which is stricter than the parent's multiple-of-2 rule.
if c.group_size != -1:
in_features = c.partition_weight_shape[0]
if c.group_size <= 0 or in_features % c.group_size != 0:
return (
False,
f"Group size ({c.group_size}) must divide input size "
f"({in_features})",
)
if not has_zentorch_op(["zentorch_woq_repack_weight", "zentorch_woq_linear"]):
return (
False,
@@ -89,8 +108,81 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
num_groups = weight_scale.shape[1]
return num_groups > 0 and in_features % num_groups == 0
def _maybe_process_da8w4_weights(self, layer: torch.nn.Module) -> bool:
"""Repack ``layer`` into the DA8W4 layout, or leave it untouched.
Returns False without mutating the layer when it is not eligible, so
the caller falls through to the W4A16 paths.
"""
if not envs.VLLM_CPU_INT4_W4A8:
return False
if not has_zentorch_op(["zentorch_dynamic_qlinear"]):
return False
# DA8W4 is symmetric-only, and the kernel rejects f32 activations.
if self.config.zero_points or self.config.weight_type == scalar_types.uint4:
return False
if self.config.act_type != torch.bfloat16:
return False
if not self._zentorch_woq_eligible(layer):
return False
weight_q = getattr(layer, self.w_q_name)
weight_s = getattr(layer, self.w_s_name)
weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q
weight_scale = weight_s.data if hasattr(weight_s, "data") else weight_s
bits = self.config.weight_type.mantissa
pack_factor = torch.iinfo(weight_packed.dtype).bits // bits
out_features, num_groups = weight_scale.shape[0], weight_scale.shape[1]
in_features = weight_packed.shape[1] * pack_factor
# AOCL sym_quant requires K/G to be a multiple of 4; K must be even to
# pack two s4 values per byte.
if (in_features // num_groups) % 4 != 0 or in_features % 2 != 0:
return False
if self.w_zp_name is not None:
setattr(layer, self.w_zp_name, None)
unpack_from_int32 = _import_unpack_from_int32()
weight_unpacked = unpack_from_int32(
weight_packed,
bits,
torch.Size([out_features, in_features]),
packed_dim=weight_q.packed_dim,
)
# The WOQ repack packs 8 int4 per int32, which on little-endian is the
# same byte stream as s4 [N, K/2]; the kernel takes either view.
layer._zentorch_da8w4_packed = (
torch.ops.zentorch.zentorch_woq_repack_weight.default(
weight_unpacked.to(torch.int8).contiguous()
).view(torch.int8)
)
# CT stores scales as [N, G]; DA8W4 wants per-group {G, N}.
layer._zentorch_da8w4_scale = weight_scale.t().to(torch.bfloat16).contiguous()
for param_name in (self.w_q_name, self.w_s_name):
param = getattr(layer, param_name, None)
if param is not None and hasattr(param, "data"):
param.data = torch.empty(0)
layer._zentorch_kind = "compressed_tensors_w4a8_da8w4"
layer._zentorch_da8w4 = True
layer._zentorch_processed_weights = True
logger.info_once(
"[zen_cpu] Using zentorch_dynamic_qlinear for DA8W4 (W4A8) "
"(weight_type=%s, group_size=%d)",
self.config.weight_type,
in_features // num_groups,
)
return True
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Repack CT GPTQ weights into the zentorch WOQ layout.
"""Repack CT GPTQ weights into the zentorch DA8W4 or WOQ layout.
Falls back to ``CPUWNA16LinearKernel.process_weights_after_loading``
via ``super()`` when the layer doesn't satisfy
@@ -101,6 +193,9 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
if getattr(layer, "_zentorch_processed_weights", False):
return
if self._maybe_process_da8w4_weights(layer):
return
if not self._zentorch_woq_eligible(layer):
logger.info_once(
"[zen_cpu] ZentorchWNA16 fast path not eligible for this "
@@ -186,6 +281,16 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if getattr(layer, "_zentorch_da8w4", False):
# The kernel reads bias through a raw pointer, so it must be
# contiguous; the weight and scales already are.
return torch.ops.zentorch.zentorch_dynamic_qlinear.default(
x if x.dtype == torch.bfloat16 else x.to(torch.bfloat16),
layer._zentorch_da8w4_packed,
layer._zentorch_da8w4_scale,
bias.contiguous() if bias is not None else None,
)
if getattr(layer, "_zentorch_processed_weights", False):
return torch.ops.zentorch.zentorch_woq_linear.default(
x,
@@ -145,16 +145,16 @@ def _block32_tiled_kernel(
for kk in range(0, K_PER_SPLIT, BLOCK_K):
if EVEN_K:
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=127)
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=127)
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
else:
k_ok = (offs_k + kk) < K
s_ok = (offs_sk + kk // 32) < K // 32
x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=127)
xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=0)
w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=127)
ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=0)
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
x_ptrs += BLOCK_K
w_ptrs += BLOCK_K
@@ -240,9 +240,9 @@ def _block32_packed_kernel(
ws_mask = ws_mask & (wg < K // 32)
x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0)
w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0)
xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=127)
xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=0)
ws = tl.load(
ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=127
ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=0
)
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
# Keep only products of matching K panels: the block diagonal.
@@ -609,6 +609,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
vllm_config = get_current_vllm_config()
parallel_config = vllm_config.parallel_config
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
@@ -784,6 +785,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
slot_mapping,
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
self.use_pcp,
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
)
assert slot_mapping is not None
if cache is not None:
@@ -1350,6 +1352,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
common_kwargs = dict(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_size,
dtype=kv_cache_dtype,
cache_dtype_str=self.kv_cache_dtype,
@@ -165,7 +165,6 @@ T = TypeVar("T", bound=SparseMLACommonMetadata)
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
metadata_cls: type[T]
require_uniform_decodes: ClassVar[bool] = False
hisparse_supports_multi_token_decode: ClassVar[bool] = False
def __init__(
self,
@@ -259,8 +258,6 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
) -> None:
if self.vllm_config.attention_config.hisparse_config is not None:
reorder_batch_threshold = 1
if not self.hisparse_supports_multi_token_decode:
supports_spec_as_decode = False
super()._init_reorder_batch_threshold(
reorder_batch_threshold,
supports_spec_as_decode,
@@ -21,6 +21,8 @@ class AttentionLayerBase(ABC):
impl: "AttentionImpl"
supports_dcp: bool = True
# Set by the speculator on its draft model's layers.
is_draft_layer: bool = False
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
"""Bind the allocated KV cache tensor to this layer.
@@ -6,6 +6,7 @@ from typing import Union
import torch
import vllm.envs as envs
from vllm.config import ParallelConfig, SchedulerConfig
from vllm.config.kernel import MoEBackend
from vllm.distributed import get_dp_group, get_pcp_group, get_tensor_model_parallel_rank
@@ -1247,6 +1248,12 @@ class FusedMoEParallelConfig:
)
# Model types validated for VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1. This flag
# also changes the MXFP4 weight shuffle layout, so using it on other models
# can produce garbled output.
_AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES = ("deepseek_v41", "deepseek_v41_text")
# Adapted from pplx-kernels tests/all_to_all_utils.py
@dataclass
class FusedMoEConfig:
@@ -1314,6 +1321,11 @@ class FusedMoEConfig:
tp_shard_with_padding: bool = False
rocm_aiter_fmoe_enabled: bool = False
aiter_fmoe_shared_expert_enabled: bool = False
# Whether to force MXFP4 (a4w4) MoE activations for DeepSeek V4.1 on
# ROCm/AITER. Opt-in via VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1; rejected
# for any other model type. Resolved here, not in the forward path,
# because get_current_vllm_config() isn't set there.
use_mxfp4_w4a4_dsv4: bool = False
def __post_init__(self):
from vllm._aiter_ops import rocm_aiter_ops
@@ -1345,6 +1357,29 @@ class FusedMoEConfig:
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if self.rocm_aiter_fmoe_enabled and envs.VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4:
from vllm.config import get_current_vllm_config_or_none
vllm_config = get_current_vllm_config_or_none()
model_type = (
getattr(vllm_config.model_config.hf_config, "model_type", None)
if vllm_config is not None
else None
)
if model_type not in _AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES:
raise ValueError(
f"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 only supports "
f"model_type in {_AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES}, "
f"got {model_type!r}. Unset this env var for this model."
)
if not rocm_aiter_ops.fused_moe_supports_quant_dtype_a():
raise ValueError(
"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 needs an AITER build "
"with fused_moe(quant_dtype_a=...) support "
"(ROCm/aiter#5439+). Upgrade AITER or unset this env var."
)
self.use_mxfp4_w4a4_dsv4 = True
if self.use_mori_kernels:
assert self.rocm_aiter_fmoe_enabled, (
"Mori needs to be used with aiter fused_moe for now."
@@ -295,6 +295,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
)
@staticmethod
@@ -344,6 +345,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
]
@staticmethod
@@ -373,11 +375,20 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
score_mode = (
"sqrtsoftplus"
if self.moe_config.routing_method == RoutingMethodType.DeepseekV4
else None
)
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
score_mode = "sqrtsoftplus"
elif routing_method == RoutingMethodType.DeepSeekV3:
# MiMo-V2.6: an ungrouped sigmoid router with a per-expert
# correction bias. aiter's flat top-k implements this natively
# (select on sigmoid(logits) + bias, return the unbiased sigmoid
# scores).
assert e_score_correction_bias is not None, (
"DeepSeekV3 routing requires e_score_correction_bias"
)
score_mode = "sigmoid"
else:
score_mode = None
return aiter_triton_kernel_w4a16_moe_forward(
hidden_states=hidden_states,
w1=w1,
@@ -51,6 +51,32 @@ from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.math_utils import round_up
logger = init_logger(__name__)
# ===========================================================================
# Weight layout
# ===========================================================================
def swigluoai_w13_interleave_perm(
experts_cls: type[mk.FusedMoEExperts],
activation: MoEActivation,
two_i: int,
device: torch.device,
) -> torch.Tensor | None:
"""Index taking a half-split w13 ``[gate | up]`` to the interleaved order
ZenDNN's ``swiglu_oai_mul`` expects, or None when no reorder is needed.
Callers apply it along whichever axis holds their output channels."""
if not getattr(experts_cls, "requires_interleaved_w13", False):
return None
if activation != MoEActivation.SWIGLUOAI:
return None
i = two_i // 2
return torch.stack(
[torch.arange(0, i, device=device), torch.arange(i, two_i, device=device)],
dim=1,
).flatten()
# ===========================================================================
# Unquantized (BF16/FP16/FP32) MoE
# ===========================================================================
@@ -1476,15 +1502,17 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
"ZenCPUExpertsInt8 requires per-channel weight scales on the layer."
)
num_experts = self.w1_scale.shape[0]
self._w1_scale_bf16 = (
# ZenDNN's fast routed MoE kernel only accepts f32 expert scales; bf16
# silently drops the layer onto the slower per-expert path.
self._w1_scale_f32 = (
self.w1_scale.detach()
.to(torch.bfloat16)
.to(torch.float32)
.reshape(num_experts, -1)
.contiguous()
)
self._w2_scale_bf16 = (
self._w2_scale_f32 = (
self.w2_scale.detach()
.to(torch.bfloat16)
.to(torch.float32)
.reshape(num_experts, -1)
.contiguous()
)
@@ -1611,8 +1639,8 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
topk_ids.to(torch.int32).contiguous(),
apply_router_weight_on_input, # skip_weighted
str(activation.value).lower(),
self._w1_scale_bf16,
self._w2_scale_bf16,
self._w1_scale_f32,
self._w2_scale_f32,
)
@@ -14,11 +14,14 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
activation_to_flashinfer_int,
quantize_nvfp4_per_token_input,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4DynamicToken,
kNvfp4Static,
)
from vllm.platforms import current_platform
@@ -40,6 +43,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
per_token_activation: bool = False,
):
super().__init__(
moe_config=moe_config,
@@ -63,6 +67,14 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
self.situ_beta = moe_config.activation_situ_beta
self.situ_linear_beta = moe_config.activation_situ_linear_beta
self.per_token_activation = per_token_activation
self.per_token_global_scale = None
if per_token_activation:
assert quant_config.a2_gscale is not None
self.per_token_global_scale = quant_config.a2_gscale.new_full(
(1,),
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
@@ -92,6 +104,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
) -> bool:
SUPPORTED_W_A = [
(kNvfp4Static, kNvfp4Dynamic),
(kNvfp4Static, kNvfp4DynamicToken),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@@ -114,6 +127,10 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
@property
def expects_unquantized_inputs(self) -> bool:
return self.per_token_activation
def workspace_shapes(
self,
M: int,
@@ -127,8 +144,8 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
workspace1 = (0,)
workspace2 = (0,)
# K is packed (K//2 for uint8), so output uses hidden_dim.
assert self.hidden_dim == K * 2
expected_hidden_dim = K if self.expects_unquantized_inputs else K * 2
assert self.hidden_dim == expected_hidden_dim
output = (M, self.hidden_dim)
return (workspace1, workspace2, output)
@@ -151,13 +168,26 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
apply_router_weight_on_input: bool | None,
):
assert self.quant_dtype == "nvfp4"
assert a1q_scale is not None
assert self.w1_scale is not None
assert self.w2_scale is not None
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
quantize_nvfp4_per_token_input(hidden_states)
)
fc2_input_scale = self.per_token_global_scale
else:
assert a1q_scale is not None
block_scale = a1q_scale
per_token_scale = None
fc2_input_scale = self.a2_gscale
assert block_scale is not None
assert fc2_input_scale is not None
# a1q_scale is (M, K//16) float8_e4m3fn from fp4_quantize.
# The functional API expects x_sf with trailing dim: (M, K//16, 1).
x_sf = a1q_scale.unsqueeze(-1)
x_sf = block_scale.unsqueeze(-1)
# The kernel defaults swiglu_{alpha,beta,limit} to the plain-SwiGLU
# values, so only forward the ones the model actually sets.
@@ -188,6 +218,9 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
}
swiglu_kwargs = {k: v for k, v in swiglu_params.items() if v is not None}
per_token_kwargs = (
{"per_token_scale": per_token_scale} if self.per_token_activation else {}
)
flashinfer_cute_dsl_fused_moe_nvfp4(
x=hidden_states,
x_sf=x_sf,
@@ -196,7 +229,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
w1_weight=w1,
w1_weight_sf=self.w1_scale,
w1_alpha=self.g1_alphas,
fc2_input_scale=self.a2_gscale,
fc2_input_scale=fc2_input_scale,
w2_weight=w2,
w2_weight_sf=self.w2_scale,
w2_alpha=self.g2_alphas,
@@ -209,4 +242,5 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
MoEActivation.SILU if activation == MoEActivation.SITU else activation
),
**swiglu_kwargs,
**per_token_kwargs,
)
@@ -375,15 +375,13 @@ def rocm_aiter_fused_experts(
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
)
# https://github.com/ROCm/aiter/pull/3123 specialized the AITER stage1 GEMMs
# for interleaved vs separated gate and up weights.
# For gpt-oss i.e. use_mxfp4_w4a16=True, the weights are shuffled by
# `rocm_aiter_ops.shuffle_weight_a16w4` in `oracle/mxfp4.py`,
# which always sets `is_guinterleave=True`.
# Hence, we pass in GateMode.INTERLEAVE to match the weight shuffling.
# AITER's stage1 GEMM needs gate_mode to match how weights were
# shuffled at load time (oracle/mxfp4.py). gpt-oss uses INTERLEAVE;
# DeepSeek V4.1 a4w4 uses SEPARATED (see below).
from aiter.ops.flydsl.moe_common import GateMode
gate_mode = ""
q_dtype_a = None
if activation == MoEActivation.SITU:
gate_mode = (
GateMode.INTERLEAVE.value
@@ -391,7 +389,15 @@ def rocm_aiter_fused_experts(
else GateMode.SEPARATED.value
)
elif quant_config.use_mxfp4_w4a16:
gate_mode = GateMode.INTERLEAVE.value
if moe_config.use_mxfp4_w4a4_dsv4:
# Opt-in a4w4 for DeepSeek V4.1 (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1).
# Must match the SEPARATED weight shuffle from load time.
from aiter import dtypes
gate_mode = GateMode.SEPARATED.value
q_dtype_a = dtypes.fp4x2
else:
gate_mode = GateMode.INTERLEAVE.value
elif activation_interleave is not None:
gate_mode = (
GateMode.INTERLEAVE.value
@@ -433,6 +439,7 @@ def rocm_aiter_fused_experts(
shared_w1_scale=shared_w1_scale,
shared_w2_scale=shared_w2_scale,
shared_expert_id=shared_expert_id,
q_dtype_a=q_dtype_a,
)
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_int,
has_flashinfer_situ_activation,
quantize_nvfp4_per_token_input,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -37,10 +38,6 @@ from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
logger = init_logger(__name__)
# Base scale for per-token NVFP4 activation quant; the kernel folds the
# per-token global scale (from the activation amax) on top of it.
_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
class TrtLlmNvFp4ExpertsBase:
"""NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface."""
@@ -259,23 +256,6 @@ class TrtLlmNvFp4ExpertsBase:
def expects_unquantized_inputs(self) -> bool:
return self.per_token_activation
def _quantize_per_token_input(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""NVFP4-quantize activations with a per-token global scale.
Returns ``(packed_fp4, block_scale, per_token_scale)``.
"""
from flashinfer import SfLayout, nvfp4_quantize
hs_fp4, hs_block_scale, per_token_scale = nvfp4_quantize(
hidden_states,
_PER_TOKEN_BASE_GLOBAL_SCALE,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
)
return hs_fp4, hs_block_scale, per_token_scale
def _get_chunk_size(self) -> int:
MAX_GRID_Y = 65535
MAX_TILE_TOKENS_DIM = 128
@@ -361,9 +341,9 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
# Per-token: input is unquantized, quantize it here. Otherwise it was
# already quantized in prepare() with the static global scale.
if self.per_token_activation:
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
self._quantize_per_token_input(hidden_states)
quantize_nvfp4_per_token_input(hidden_states)
)
else:
block_scale, per_token_scale = a1q_scale, None
@@ -437,7 +417,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
) -> UnfinalizedMoEOutput | None:
assert self._supports_activation(activation)
# Per-token defers input quant to _invoke_kernel, so a1q_scale is None.
assert a1q_scale is not None or self.per_token_activation
assert a1q_scale is not None or self.expects_unquantized_inputs
# DeepEP produces int64 indexes.
topk_ids = topk_ids.to(dtype=torch.int32)
@@ -539,7 +519,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
import flashinfer
assert self._supports_activation(activation)
assert a1q_scale is not None or self.per_token_activation
assert a1q_scale is not None or self.expects_unquantized_inputs
assert self.quant_config.w1_scale is not None
assert self.quant_config.w2_scale is not None
assert (
@@ -551,9 +531,9 @@ class TrtLlmNvFp4ExpertsMonolithic(
)
# Per-token: input is unquantized, quantize it here (see modular apply).
if self.per_token_activation:
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
self._quantize_per_token_input(hidden_states)
quantize_nvfp4_per_token_input(hidden_states)
)
else:
block_scale, per_token_scale = a1q_scale, None
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zentorch fused MoE experts for AMD Zen CPUs."""
from collections.abc import Callable
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
_moe_activation_to_str,
has_zentorch_op,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import select_experts
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
)
class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
"""DA8W4 (W4A8) group-quantized monolithic MoE experts."""
# swiglu_oai_mul reads gate/up interleaved, not in the half-split order the
# weight loader leaves behind.
requires_interleaved_w13 = True
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
# Defaults hold until process_weights_after_loading sees the layer.
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
)
self.custom_routing_function: Callable | None = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Capture the router config that the monolithic apply() signature
cannot carry; Custom routing has no other source for it."""
self.renormalize = layer.renormalize
self.custom_routing_function = layer.custom_routing_function
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return has_zentorch_op(["zentorch_fused_moe", "zentorch_woq_repack_weight"])
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return supported, reason
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
# The grouped GEMM needs tokens spread over at least two experts, which
# top_k=1 cannot satisfy once decode drops to a single token.
if moe_config.experts_per_token < 2:
return False, "kernel requires experts_per_token >= 2"
return True, None
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return _moe_activation_to_str(activation) in _ZENTORCH_MOE_ACTIVATIONS
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
# apply() passes select_experts' global ids straight to the kernel, so
# every expert has to be resident on the rank.
return not moe_parallel_config.use_ep
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) in [
(kInt4Static, None),
(kInt4Static32, None),
]
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.Custom,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
"ZentorchExpertsInt4 does not support "
"apply_router_weight_on_input=True."
)
if (
self.moe_config.routing_method == RoutingMethodType.Custom
and self.custom_routing_function is None
):
raise RuntimeError(
"ZentorchExpertsInt4 needs the model's custom_routing_function "
"for RoutingMethodType.Custom."
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
custom_routing_function=self.custom_routing_function,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
output = torch.empty_like(hidden_states)
torch.ops.zentorch.zentorch_fused_moe(
output,
hidden_states,
w1,
w2,
self.w1_bias,
self.w2_bias,
topk_weights,
topk_ids,
False, # skip_weighted
_moe_activation_to_str(activation),
self.w1_scale,
self.w2_scale,
)
return output
__all__ = ["ZentorchExpertsInt4"]
@@ -12,6 +12,7 @@ from compressed_tensors.quantization import (
import vllm._custom_ops as ops
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config.kernel import MoEBackend
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
@@ -55,6 +56,7 @@ class WNA16MoEBackend(Enum):
BATCHED_MARLIN = "BATCHED_MARLIN"
HUMMING = "HUMMING"
CPU = "CPU"
ZEN_CPU = "ZEN_CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
TRITON = "TRITON"
XPU = "XPU"
@@ -98,6 +100,12 @@ def backend_to_kernel_cls(
)
return [CPUExpertsInt4]
elif backend == WNA16MoEBackend.ZEN_CPU:
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
return [ZentorchExpertsInt4]
elif backend == WNA16MoEBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
Int4EmulationTritonExperts,
@@ -117,7 +125,7 @@ def backend_to_kernel_cls(
def _get_priority_backends() -> list[WNA16MoEBackend]:
"""Get available backends in priority order based on platform and config."""
if current_platform.is_cpu():
return [WNA16MoEBackend.CPU]
return [WNA16MoEBackend.ZEN_CPU, WNA16MoEBackend.CPU]
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
@@ -160,6 +168,27 @@ def _backend_incompatibility_reason(
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
if backend == WNA16MoEBackend.ZEN_CPU:
if not envs.VLLM_CPU_INT4_W4A8:
return "VLLM_CPU_INT4_W4A8=0 disables the DA8W4 path"
if may_have_zp:
return "zero points are not supported"
if isinstance(quant_config, MoeWNA16Config):
return "the MoeWNA16 weight layout is not supported"
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
return "GPTQ activation ordering is not supported"
if (
isinstance(quant_config, QuantizationArgs)
and quant_config.actorder == "group"
):
return "group activation ordering is not supported"
group_size = getattr(quant_config, "group_size", None)
if group_size is None or group_size <= 0:
return "DA8W4 requires group-quantized weights"
# AOCL sym_quant requires the group size to be a multiple of 4.
if group_size % 4 != 0:
return f"group size {group_size} is not a multiple of 4"
if backend == WNA16MoEBackend.TRITON:
if may_have_bias:
return "expert bias is not supported"
@@ -403,10 +432,14 @@ def make_wna16_moe_kernel(
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsWNA16,
)
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming
# grouped/indexed experts, and Int4EmulationTritonExperts
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4,
# ZentorchExpertsInt4, the Humming grouped/indexed experts, and
# Int4EmulationTritonExperts
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
MarlinExperts,
BatchedMarlinExperts,
@@ -414,6 +447,7 @@ def make_wna16_moe_kernel(
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
ZentorchExpertsInt4,
Int4EmulationTritonExperts,
Rdna3WNA16Experts,
)
@@ -1034,6 +1068,64 @@ def _process_weights_cpu(
)
def _zen_repack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
"""Repack CT int4 ``[E, N, K//8]`` into zentorch s4 ``[E, N, K//8]``."""
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
_import_unpack_from_int32,
)
num_experts, out_features, _ = packed.shape
# packed_dim indexes the 2D expert slice, which is N-first, so K is dim 1.
unpacked = _import_unpack_from_int32()(
packed,
4,
torch.Size([num_experts, out_features, in_features]),
packed_dim=1,
)
# CT is already N-first, which is what the repack consumes.
repack_op = torch.ops.zentorch.zentorch_woq_repack_weight.default
return torch.stack([repack_op(w) for w in unpacked])
def _process_weights_zen_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, # w13_qweight
torch.Tensor, # w2_qweight
torch.Tensor, # w13_scales
torch.Tensor, # w2_scales
torch.Tensor | None, # w13_qzeros
torch.Tensor | None, # w2_qzeros
torch.Tensor | None, # w13_input_global_scale
torch.Tensor | None, # w2_input_global_scale
torch.Tensor | None, # w13_bias
torch.Tensor | None, # w2_bias
]:
"""Zen CPU INT4 DA8W4 (W4A8) weight post-processing."""
# CT WNA16 buffers are N-first, so the scales are [E, N, G].
hidden_size = w2_scale.shape[1]
intermediate_size = w13_scale.shape[1] // 2
# The kernel reads per-group scales as [E, G, N], so N-first needs a swap.
return (
_zen_repack_s4(w13.data, hidden_size),
_zen_repack_s4(w2.data, intermediate_size),
w13_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
w2_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
None, # w13_qzeros (symmetric)
None, # w2_qzeros (symmetric)
None, # w13_input_global_scale
None, # w2_input_global_scale
w13_bias,
w2_bias,
)
def _process_weights_xpu(
layer: torch.nn.Module,
quant_config: QuantizationConfig,
@@ -1584,6 +1676,15 @@ def convert_to_wna16_moe_kernel_format(
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.ZEN_CPU:
return _process_weights_zen_cpu(
w13,
w2,
w13_scale,
w2_scale,
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
return _process_weights_flashinfer(
w13,
@@ -1382,6 +1382,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias: torch.Tensor | None = None,
_cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None,
activation: MoEActivation | None = None,
use_separated_a4w4: bool = False,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -1626,10 +1627,16 @@ def convert_weight_to_mxfp4_moe_kernel_format(
from aiter.ops.shuffle import shuffle_scale as _shuf_s
from aiter.ops.shuffle import shuffle_weight as _shuf_w
# DeepSeek V4.1 a4w4 uses ATOM's SEPARATED gate/up layout instead of
# the default INTERLEAVE shuffle (INTERLEAVE + fp4x2 has no tuned
# kernel and produces garbage output). Must match GateMode.SEPARATED
# in rocm_aiter_moe.py.
is_guinterleave = not use_separated_a4w4
w13_weight = torch.nn.Parameter(
_shuf_w(
w13_weight.data.view(torch.float4_e2m1fn_x2),
is_guinterleave=True,
is_guinterleave=is_guinterleave,
gate_up=True,
),
requires_grad=False,
@@ -1637,14 +1644,14 @@ def convert_weight_to_mxfp4_moe_kernel_format(
shuffled_w13_scale = _shuf_s(
w13_weight_scale.reshape(-1, w13_weight_scale.shape[-1]),
num_experts,
True,
is_guinterleave,
True,
)
w2_weight = torch.nn.Parameter(
_shuf_w(
w2_weight.data.view(torch.float4_e2m1fn_x2),
is_guinterleave=True,
is_guinterleave=is_guinterleave,
gate_up=False,
),
requires_grad=False,
@@ -1653,7 +1660,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
shuffled_w2_scale = _shuf_s(
w2_weight_scale.reshape(-1, w2_weight_scale.shape[-1]),
num_experts,
True,
is_guinterleave,
False,
)
@@ -612,7 +612,14 @@ def make_nvfp4_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM and per_token_activation:
if (
backend
in (
NvFp4MoeBackend.FLASHINFER_TRTLLM,
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
)
and per_token_activation
):
extra_kwargs["per_token_activation"] = True
# Create Experts.
@@ -371,6 +371,16 @@ def select_unquantized_moe_backend(
)
def aiter_moe_intermediate_alignment(intermediate: int) -> int:
"""Intermediate-size alignment required by AITER's CK 2stages MoE kernel.
AITER dispatches on ``inter_dim <= 192``: below the threshold both stages
use 64-wide tiles, above it at least one stage uses a 128-wide tile, and
CK's ``IsSupportedArgument`` rejects a size not divisible by that width.
"""
return 64 if intermediate <= 192 else 128
def unquantized_round_up_hidden_size_and_intermediate_size(
backend: UnquantizedMoeBackend,
hidden_size: int,
@@ -379,9 +389,30 @@ def unquantized_round_up_hidden_size_and_intermediate_size(
"""Round up dimensions before allocation to satisfy the selected kernel."""
if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
intermediate_size = round_up(intermediate_size, 128)
elif backend == UnquantizedMoeBackend.AITER:
intermediate_size = round_up(
intermediate_size, aiter_moe_intermediate_alignment(intermediate_size)
)
return hidden_size, intermediate_size
def _zero_intermediate_padding(
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
) -> None:
"""Zero intermediate padding; reloads rewrite only the checkpoint slices."""
unpadded = moe_config.intermediate_size_per_partition_unpadded
assert unpadded is not None
padded = moe_config.intermediate_size_per_partition
if padded <= unpadded:
return
w13_weight[:, unpadded:padded].zero_()
if moe_config.is_act_and_mul:
w13_weight[:, padded + unpadded :].zero_()
w2_weight[:, :, unpadded:].zero_()
def convert_to_unquantized_kernel_format(
unquantized_backend: UnquantizedMoeBackend,
moe_config: FusedMoEConfig,
@@ -424,6 +455,7 @@ def convert_to_unquantized_kernel_format(
return layout.full_gate_weight, layout.full_down_weight
if unquantized_backend == UnquantizedMoeBackend.AITER:
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
w13_weight.is_shuffled = True
w2_weight.is_shuffled = True
@@ -445,15 +477,7 @@ def convert_to_unquantized_kernel_format(
)
moe_config.intermediate_size_per_partition = padded_intermediate
# Reloads only overwrite checkpoint slices. An earlier in-place
# permutation can leave nonzero values in the raw padding slots.
unpadded = moe_config.intermediate_size_per_partition_unpadded
assert unpadded is not None
if padded_intermediate > unpadded:
w13_weight[:, unpadded:padded_intermediate].zero_()
if is_act_and_mul:
w13_weight[:, padded_intermediate + unpadded :].zero_()
w2_weight[:, :, unpadded:].zero_()
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
convert_moe_weights_to_flashinfer_trtllm_block_layout(
@@ -27,6 +27,7 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
self.num_dispatchers_ = num_dispatchers
self.max_tokens_per_rank = max_tokens_per_rank
self.use_fp8_dispatch = use_fp8_dispatch
self._dispatch_topk_ids: torch.Tensor | None = None
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
@@ -92,6 +93,12 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
# mori expects one scale slot per token; broadcast.
scale = scale.expand(a1.shape[0], 1).contiguous()
# mori's combine() reduces over this rank's own [num_tokens, topk]
# routing. The modular kernel rebinds topk_ids to the dispatched ids
# returned below before it calls finalize(), so hold a reference to
# the pre-dispatch tensor (ROCm/mori#475).
self._dispatch_topk_ids = topk_ids
(
dispatch_a1,
dispatch_weights,
@@ -122,9 +129,13 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> None:
num_token = output.shape[0]
assert self._dispatch_topk_ids is not None, (
"finalize() called before prepare(); mori combine() needs the "
"routing tensor that dispatch() was given"
)
result = self.mori_op.combine(
fused_expert_output,
None,
topk_ids,
self._dispatch_topk_ids,
)[0]
output.copy_(result[:num_token])
@@ -691,7 +691,10 @@ class RoutedExperts(PluggableLayer):
)
and is_transposed
):
loaded_weight = loaded_weight.t().contiguous()
# Transpose on the parameter's device: a CPU transpose of the
# mmap view reads the file column-wise, which is very slow on
# NFS. The contiguous .to() reads it sequentially.
loaded_weight = loaded_weight.to(param.device).t().contiguous()
if shard_id not in ("w1", "w2", "w3"):
raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.")
@@ -19,6 +19,11 @@ from vllm.model_executor.layers.fused_moe.router.dsv4_topk import (
dsv4_topk,
)
# AITER instantiates biased_grouped_topk only for these NUM_GRP values, with at
# most this many experts per group.
AITER_SUPPORTED_NUM_GRP = (1, 2, 4, 8)
AITER_MAX_EXPERTS_PER_GROUP = 32
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
@@ -104,17 +109,40 @@ def vllm_topk_softplus_sqrt(
@functools.lru_cache(maxsize=8)
def _aiter_get_num_expert_group(num_experts: int) -> int:
_AITER_MAX_EXPERTS_PER_GROUP = 32
g = max(1, -(-num_experts // _AITER_MAX_EXPERTS_PER_GROUP))
g = max(1, -(-num_experts // AITER_MAX_EXPERTS_PER_GROUP))
while num_experts % g != 0:
g += 1
assert num_experts % g == 0, f"{num_experts=} not divisible by {g=}"
assert num_experts // g <= _AITER_MAX_EXPERTS_PER_GROUP, (
f"group size {num_experts // g} exceeds limit {_AITER_MAX_EXPERTS_PER_GROUP}"
# Grouping is a no-op here (topk_group == num_expert_group), so any divisor
# within the limit routes identically and we are free to pick one AITER has
# a kernel for. If none fits, the unsupported value is returned as-is --
# callers must gate on _aiter_can_use_biased_grouped_topk.
if g not in AITER_SUPPORTED_NUM_GRP:
g = next(
(
c
for c in sorted(AITER_SUPPORTED_NUM_GRP, reverse=True)
if num_experts % c == 0
and num_experts // c <= AITER_MAX_EXPERTS_PER_GROUP
),
g,
)
assert num_experts // g <= AITER_MAX_EXPERTS_PER_GROUP, (
f"group size {num_experts // g} exceeds limit {AITER_MAX_EXPERTS_PER_GROUP}"
)
return g
def _aiter_can_use_biased_grouped_topk(num_experts: int, topk: int) -> bool:
"""Whether AITER has an instantiated kernel for this routing shape.
Both checks are needed: an unsupported group count can still be small
enough to satisfy ``topk >= g`` (num_experts=33 gives g=3).
"""
g = _aiter_get_num_expert_group(num_experts)
return topk >= g and g in AITER_SUPPORTED_NUM_GRP
def fused_topk_bias(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
@@ -229,7 +257,7 @@ def fused_topk_bias(
M = hidden_states.size(0)
num_experts = gating_output.shape[-1]
num_expert_group = _aiter_get_num_expert_group(num_experts)
if topk >= num_expert_group:
if _aiter_can_use_biased_grouped_topk(num_experts, topk):
topk_weights = torch.empty(
M, topk, dtype=torch.float32, device=hidden_states.device
)
@@ -720,7 +720,7 @@ class MoERunner(MoERunnerInterface):
)
# Record before `_maybe_pad_hidden_states` pads activations to match
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
# `moe_config.hidden_dim`, e.g. after `align_fp4_moe_hidden_dim_for_fi`
# so routed output can be trimmed before
# shared+routed add / latent up proj if needed.
@@ -147,6 +147,11 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams:
rescale_factor)`` from the processor config."""
config = get_processor_config(model_config.model, revision=model_config.revision)
config = config.get("image_processor", config)
if not any(
key in config
for key in ("do_normalize", "do_rescale", "image_mean", "image_std")
):
config = model_config.hf_image_processor_config
has_norm_params = "image_mean" in config and "image_std" in config
do_normalize = bool(config.get("do_normalize", has_norm_params))
@@ -241,6 +246,10 @@ class FusedMMInputNorm(CustomOp):
self.register_buffer("weight", (rescale_factor / std).to(device))
self.register_buffer("bias", (-mean / std).to(device))
@classmethod
def enabled(cls) -> bool:
return True
@property
def input_dtype(self) -> torch.dtype | None:
return torch.uint8
+73 -16
View File
@@ -211,6 +211,79 @@ class GemmaRMSNorm(CustomOp):
return out
@CustomOp.register("layer_norm")
class LayerNorm(CustomOp):
"""Standard (mean-centered) LayerNorm.
Drop-in for a bare `nn.LayerNorm` that dispatches to a fused XPU kernel
when one is available, and to the native implementation otherwise.
Normalization runs in the wider of the input and parameter dtypes and the
result is cast back to the input dtype, so `dtype=torch.float32` gives an
fp32 reduction for a lower-precision input.
"""
def __init__(
self,
hidden_size: int,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
weight_dtype = dtype or torch.get_default_dtype()
self.weight: nn.Parameter | None = None
self.bias: nn.Parameter | None = None
if elementwise_affine:
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype))
if bias:
self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype))
def _compute_dtype(self, x: torch.Tensor) -> torch.dtype:
if self.weight is None:
return x.dtype
return torch.promote_types(x.dtype, self.weight.dtype)
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
compute_dtype = self._compute_dtype(x)
if compute_dtype != x.dtype:
return F.layer_norm(
x.to(compute_dtype),
self.normalized_shape,
self.weight,
self.bias,
self.eps,
).type_as(x)
return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_layer_norm
if (
self.weight is None
or self.bias is None
or not hasattr(torch.ops._C, "layer_norm")
# The kernel reduces in the input dtype, so a widened reduction
# takes the native path.
or self._compute_dtype(x) != x.dtype
):
return self.forward_native(x)
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_layer_norm(out, x, self.weight, self.bias, self.eps)
return out
def extra_repr(self) -> str:
return f"hidden_size={self.normalized_shape[0]}, eps={self.eps}"
# --8<-- [start:rms_norm_gated]
@CustomOp.register("rms_norm_gated")
class RMSNormGated(CustomOp):
@@ -350,19 +423,3 @@ class RMSNormGated(CustomOp):
self, x: torch.Tensor, z: torch.Tensor | None = None
) -> torch.Tensor:
return self.forward_cuda(x, z)
class LayerNorm(nn.Module):
"""Layer Normalization."""
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.dim = dim
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32))
self.bias = nn.Parameter(torch.zeros(dim, dtype=torch.float32))
def forward(self, x: torch.Tensor):
return F.layer_norm(
x.float(), (self.dim,), self.weight, self.bias, self.eps
).type_as(x)
+31 -10
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
from dataclasses import dataclass, replace
import torch
@@ -58,10 +58,33 @@ def compute_mamba_prefill_checkpoints(
return offsets, cols
def _gather_checkpoint_state_indices(
block_table: torch.Tensor, request_rows: torch.Tensor, block_cols: torch.Tensor
) -> torch.Tensor:
return torch.where(
block_cols >= 0, block_table[request_rows, block_cols], NULL_BLOCK_ID
)
@dataclass
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
block_cols: torch.Tensor | None = None
def regather_state_indices(
self, block_table: torch.Tensor
) -> "MambaPrefillCheckpointMetadata":
assert self.request_rows is not None and self.block_cols is not None
return replace(
self,
state_indices=_gather_checkpoint_state_indices(
block_table, self.request_rows, self.block_cols
),
)
class MambaPrefillCheckpointBuilder:
@@ -76,6 +99,8 @@ class MambaPrefillCheckpointBuilder:
m: CommonAttentionMetadata,
request_rows: list[int],
) -> MambaPrefillCheckpointMetadata | None:
if self.vllm_config.cache_config.mamba_cache_mode != "align":
return None
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
return None
assert m.seq_lens_cpu_upper_bound is not None
@@ -109,17 +134,13 @@ class MambaPrefillCheckpointBuilder:
checkpoint_cols_tensor = async_tensor_h2d(
checkpoint_cols, dtype=torch.int64, device=m.query_start_loc.device
)
checkpoint_state_indices = m.block_table_tensor[
request_rows_tensor, checkpoint_cols_tensor
]
checkpoint_state_indices = torch.where(
checkpoint_cols_tensor >= 0,
checkpoint_state_indices,
NULL_BLOCK_ID,
)
return MambaPrefillCheckpointMetadata(
checkpoint_offsets_tensor,
checkpoint_state_indices,
_gather_checkpoint_state_indices(
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
@@ -260,8 +260,6 @@ class MambaMixer(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -297,8 +295,6 @@ class MambaMixer(MambaBase, PluggableLayer):
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
num_decode_tokens = attn_metadata.num_decode_tokens
num_prefills = attn_metadata.num_prefills # request count
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
has_prefill = num_prefill_tokens > 0
has_decode = num_decode_tokens > 0
num_actual_tokens = num_prefill_tokens + num_decode_tokens
@@ -314,34 +310,6 @@ class MambaMixer(MambaBase, PluggableLayer):
gate_p = prefill_decode_split.gate_p
gate_d = prefill_decode_split.gate_d
if is_mamba_cache_all:
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_d = None
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
num_computed_tokens_p = None
ssm_outputs = []
if has_prefill:
@@ -355,11 +323,6 @@ class MambaMixer(MambaBase, PluggableLayer):
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
query_start_loc=query_start_loc_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
@@ -383,10 +346,6 @@ class MambaMixer(MambaBase, PluggableLayer):
cache_indices=state_indices_tensor_p,
has_initial_state=has_initial_states_p,
query_start_loc=query_start_loc_p,
block_size=mamba_block_size,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
cu_chunk_seqlen=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
)
@@ -396,16 +355,6 @@ class MambaMixer(MambaBase, PluggableLayer):
# state_indices_tensor_d is assigned when attn_metadata is not None,
# and has_decode is only True when attn_metadata is not None
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
# 2. Convolution sequence transformation
conv_out_d = causal_conv1d_update(
hidden_states_BC_d.transpose(0, 1),
@@ -414,8 +363,6 @@ class MambaMixer(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
).transpose(0, 1)
# 3. State Space Model sequence transformation.
@@ -437,8 +384,8 @@ class MambaMixer(MambaBase, PluggableLayer):
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=scan_outputs_d,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
+126 -198
View File
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
commit_replayssm_ring_trackers,
reset_replayssm_ring_trackers,
selective_state_update,
selective_state_update_replayssm_flashinfer,
@@ -69,6 +70,13 @@ from vllm.v1.kv_cache_interface import KVCacheGroupSpec
logger = init_logger(__name__)
def _view_mtp_decode_tensor(
tensor: torch.Tensor, decode_batch: int, spec_query_len: int
) -> torch.Tensor:
return tensor.view(decode_batch, spec_query_len, *tensor.shape[1:])
# Added by the IBM Team, 2024
@@ -232,13 +240,14 @@ def mamba_v2_sharded_weight_loader(
# - the ignore is for a mundane mypy error as it does not
# seem to handle slices well.
# https://github.com/python/mypy/issues/2410
target_slice = param.data[boundary : (boundary + take), ...]
param.data[
boundary : (boundary + take), ... # type: ignore[misc]
] = loaded_weight[
loaded_start_idx : (
loaded_start_idx + take
) # type: ignore[misc]
] # type: ignore[misc]
].view_as(target_slice)
# move indexing boundaries
boundary += shard_size
@@ -440,17 +449,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
tp_rank,
)
# Apply the custom weight loader to in_proj.weight
# Apply the custom weight loader to in_proj.weight and its scales.
# Works for both non-quantized (Parameter) and quantized
# (ModelWeightParameter which extends BasevLLMParameter)
if isinstance(self.in_proj.weight, BasevLLMParameter):
# For BasevLLMParameter subclasses (quantized layers like FP8)
# These have a weight_loader property that can be directly set
self.in_proj.weight.weight_loader = mamba_loader
else:
# For standard Parameter (non-quantized layers)
delattr(self.in_proj.weight, "weight_loader")
set_weight_attrs(self.in_proj.weight, {"weight_loader": mamba_loader})
for attr_name in ("weight", "weight_scale", "input_scale"):
param = getattr(self.in_proj, attr_name, None)
if param is not None:
if isinstance(param, BasevLLMParameter):
param.weight_loader = mamba_loader
else:
if hasattr(param, "weight_loader"):
delattr(param, "weight_loader")
set_weight_attrs(param, {"weight_loader": mamba_loader})
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
@@ -534,15 +544,11 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.kv_cache = tuple(torch.tensor([]) for _ in range(_n_state))
self._replayssm_ring_start = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_query_len = torch.empty(0, dtype=torch.int32)
self._commits_replayssm_trackers = True
self._updates_replayssm_trackers = True
self.num_spec = vllm_config.num_speculative_tokens
if self.num_spec > 0:
self.register_buffer(
"_decode_state_offsets",
torch.arange(1 + self.num_spec, dtype=torch.int32).unsqueeze(0),
persistent=False,
)
# Pre-compute sizes for forward pass
self.tped_intermediate_size = self.intermediate_size // self.tp_size
@@ -720,9 +726,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
ring_start = prev_num_accepted = None
use_spec_decode = self.num_spec > 0
ring_start = prev_num_accepted = prev_query_len = None
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -744,6 +749,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
ring_start = self._replayssm_ring_start
prev_num_accepted = self._replayssm_prev_num_accepted
prev_query_len = self._replayssm_prev_query_len
else:
x_cache = dt_cache = B_cache = None
has_initial_states_p = attn_metadata.has_initial_states_p
@@ -755,6 +761,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
replayssm_state_indices_d = attn_metadata.replayssm_state_indices_d
num_accepted_tokens = attn_metadata.num_accepted_tokens
query_start_loc_d = attn_metadata.query_start_loc_d
num_decodes = attn_metadata.num_decodes
@@ -788,45 +795,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
dim=0,
)
if is_mamba_cache_all:
# If prefix caching is enabled, retrieve the relevant variables
# for prefill and decode
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
if attn_metadata.block_idx_last_scheduled_token_prev_step is not None:
block_idx_last_scheduled_token_prev_step_d, _ = torch.split(
attn_metadata.block_idx_last_scheduled_token_prev_step,
[num_decodes, num_prefills],
dim=0,
)
else:
block_idx_last_scheduled_token_prev_step_d = None
# Prefill-only variables:
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_computed_token_d = None
block_idx_last_scheduled_token_prev_step_d = None
num_computed_tokens_p = None
preallocated_ssm_out_d, preallocated_ssm_out_p = torch.split(
output[:num_actual_tokens],
[num_decode_tokens, num_prefill_tokens],
@@ -843,11 +811,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
# to by "state_indices_tensor_p".
# In particular, it will always write the state at the
# sequence end.
# In addition, "block_idx_first_scheduled_token_p" and
# "block_idx_last_scheduled_token_p"
# are provided (which are pointers into
# "state_indices_tensor_p"), it will write additional cache
# states aligned at "block_size_to_align".
x = hidden_states_B_C_p.transpose(
0, 1
) # this is the form that causal-conv see
@@ -859,11 +822,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
query_start_loc=query_start_loc_p,
).transpose(0, 1)[:num_prefill_tokens]
@@ -876,14 +834,9 @@ class MambaMixer2(MambaBase, PluggableLayer):
initial_states = None
if has_initial_states_p is not None and prep_initial_states:
assert state_indices_tensor_p is not None
kernel_ssm_indices = state_indices_tensor_p
if is_mamba_cache_all:
kernel_ssm_indices = state_indices_tensor_p.gather(
1, block_idx_last_computed_token_p.unsqueeze(1)
).squeeze(1)
initial_states = torch.where(
has_initial_states_p[:, None, None, None],
ssm_state[kernel_ssm_indices],
ssm_state[state_indices_tensor_p],
0,
)
@@ -906,128 +859,56 @@ class MambaMixer2(MambaBase, PluggableLayer):
cu_chunk_seqlens=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
initial_states=initial_states,
return_intermediate_states=is_mamba_cache_all,
dt_softplus=True,
dt_limit=(0.0, float("inf")),
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
state_dtype=ssm_state.dtype,
)
if is_mamba_cache_all:
assert mamba_block_size is not None
assert state_indices_tensor_p is not None
assert block_idx_first_scheduled_token_p is not None
assert block_idx_last_scheduled_token_p is not None
assert last_chunk_indices_p is not None
assert num_computed_tokens_p is not None
# The chunk_stride is the number of chunks per mamba block
# e.g., if mamba_block_size = 512 and chunk_size = 256,
# then chunk_stride = 2
chunk_stride = mamba_block_size // chunk_size
# Save state for sequences with more than just final state,
# batched over sequences so that the number of kernel launches
# does not scale with the number of prefills.
n_blocks_to_fill = (
block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p
).clamp(min=0)
total_fill = attn_metadata.num_state_writes_p
if total_fill > 0:
first_chunk = torch.zeros_like(last_chunk_indices_p)
first_chunk[1:] = last_chunk_indices_p[:-1] + 1
# The chunk that completes this sequence's first mamba block.
# A block spans chunk_stride chunks. When the sequence resumed
# mid-block, the chunks before this step are not in varlen_states,
# so subtract them: the boundary sits earlier by their count.
num_unaligned_computed_tokens = (
num_computed_tokens_p % mamba_block_size
)
unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size
first_aligned_chunk = (
first_chunk + (chunk_stride - 1) - unaligned_chunk_shift
)
# Flatten all writes into one list: fill_counts=[2,1,2]
# -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1]
fill_counts = n_blocks_to_fill.long()
# output_size, or repeat_interleave reads the size off the device
seq_ids = torch.repeat_interleave(
torch.arange(num_prefills, device=ssm_state.device),
fill_counts,
output_size=total_fill,
)
write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts
write_offsets = torch.arange(
total_fill, device=ssm_state.device
) - torch.repeat_interleave(
write_starts, fill_counts, output_size=total_fill
)
cache_blocks_to_fill = state_indices_tensor_p[
seq_ids,
block_idx_first_scheduled_token_p[seq_ids] + write_offsets,
]
# Write the states. Each row goes to its own cache block,
# so the indexed copy has no duplicate destinations.
ssm_state[cache_blocks_to_fill] = varlen_states[
first_aligned_chunk[seq_ids] + write_offsets * chunk_stride
]
# For all seqs, store the last state (note: might be partial):
assert state_indices_tensor_p is not None
ssm_state[
state_indices_tensor_p.gather(
1, block_idx_last_scheduled_token_p.unsqueeze(1)
).squeeze(1)
] = varlen_states[last_chunk_indices_p]
else:
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_indices_tensor_p,
)
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_indices_tensor_p,
)
# Process decode requests
if has_decode:
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
if self.num_spec > 0:
assert block_idx_last_scheduled_token_prev_step_d is not None
input_indices = (
block_idx_last_scheduled_token_prev_step_d.unsqueeze(1)
+ self._decode_state_offsets
)
output_indices = (
block_idx_last_scheduled_token_d.unsqueeze(1)
+ self._decode_state_offsets
)
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, input_indices
)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, output_indices
)
else:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
# Without caching, read and write in-place to the same blocks:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
if (
self.use_replayssm
and self.mamba_config.backend == MambaBackendEnum.FLASHINFER
and use_spec_decode
and self._commits_replayssm_trackers
):
assert ring_start is not None
assert prev_num_accepted is not None
assert prev_query_len is not None
assert replayssm_state_indices_d is not None
assert num_accepted_tokens is not None
assert query_start_loc_d is not None
assert x_cache is not None
assert self.replayssm_buffer_len is not None
# The previous MTP query's accepted prefix is known only now.
# Commit it once, before this cache group's layers evaluate the
# current speculative query with the shared ring positions.
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
replayssm_state_indices_d,
num_accepted_tokens,
query_start_loc_d,
logical_window=self.replayssm_buffer_len,
ring_buffer_len=x_cache.size(2),
)
# 2. Convolution sequence transformation
hidden_states_B_C_d = causal_conv1d_update(
@@ -1037,11 +918,15 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
max_query_len=state_indices_tensor_d.size(-1),
# ReplaySSM keeps one physical state block while a speculative
# decode call still processes the full target + draft window.
max_query_len=(
1 + self.num_spec
if self.use_replayssm and use_spec_decode
else state_indices_tensor_d.size(-1)
),
)
hidden_states_d, B_d, C_d = self.split_hidden_states_B_C_fn(
@@ -1077,7 +962,35 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
assert ring_start is not None
assert prev_num_accepted is not None
assert prev_query_len is not None
assert attn_metadata.replayssm_scratch is not None
fi_cu_seqlens = query_start_loc_d
fi_max_seqlen = None
if use_spec_decode:
spec_query_len = 1 + self.num_spec
fi_max_seqlen = spec_query_len
assert replayssm_state_indices_d is not None
decode_batch = replayssm_state_indices_d.size(0)
if num_decode_tokens == decode_batch * spec_query_len:
hidden_states_d = _view_mtp_decode_tensor(
hidden_states_d, decode_batch, spec_query_len
)
dt_d = _view_mtp_decode_tensor(
dt_d, decode_batch, spec_query_len
)
B_d = _view_mtp_decode_tensor(
B_d, decode_batch, spec_query_len
)
C_d = _view_mtp_decode_tensor(
C_d, decode_batch, spec_query_len
)
preallocated_ssm_out_d = _view_mtp_decode_tensor(
preallocated_ssm_out_d,
decode_batch,
spec_query_len,
)
fi_cu_seqlens = None
fi_max_seqlen = None
selective_state_update_replayssm_flashinfer(
ssm_state,
hidden_states_d,
@@ -1091,19 +1004,25 @@ class MambaMixer2(MambaBase, PluggableLayer):
dt_cache,
ring_start,
prev_num_accepted,
prev_query_len,
logical_window=self.replayssm_buffer_len,
D=D_d,
dt_bias=dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=replayssm_state_indices_d,
scratch=attn_metadata.replayssm_scratch,
update_trackers=self._updates_replayssm_trackers,
update_trackers=(
self._updates_replayssm_trackers and not use_spec_decode
),
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
),
stochastic_rounding_philox_rounds=(
self.mamba_config.stochastic_rounding_philox_rounds
),
cu_seqlens=fi_cu_seqlens,
max_seqlen=fi_max_seqlen,
enable_pdl=False,
)
else:
selective_state_update_replayssm_output_only(
@@ -1123,7 +1042,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
write_pos=attn_metadata.write_pos_d,
is_flush=attn_metadata.is_flush_d,
max_cache_len=self.replayssm_buffer_len,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
@@ -1143,8 +1062,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
D_d,
dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=query_start_loc_d,
@@ -1185,6 +1104,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
tp_world_size=tp_world_size,
logical_window=self.replayssm_buffer_len,
backend=self.mamba_config.backend,
num_speculative_tokens=self.num_spec,
)
return base_shape
@@ -1202,9 +1122,10 @@ def share_replayssm_ring_trackers(
Layers backed by one KV-cache group use the same physical block indices and
can therefore share cursors. Different KV-cache groups may assign different
block indices to the same request and must keep separate cursor tensors.
The final local layer in each group advances its cursors after every layer
in that group has consumed the previous values.
block indices to the same request and must keep separate cursor tensors. For
speculative decode, the first local layer commits the preceding acceptance;
for standard decode, the final local layer advances after all layers consume
the previous values.
"""
replayssm_mixers: dict[str, MambaMixer2] = {}
for layer_name in ordered_layer_names:
@@ -1230,6 +1151,7 @@ def share_replayssm_ring_trackers(
groups_by_namespace.setdefault(namespace, []).append(layer_name)
for group_layer_names in groups_by_namespace.values():
first_layer_name = group_layer_names[0]
last_layer_name = group_layer_names[-1]
first_mixer = replayssm_mixers[group_layer_names[0]]
@@ -1244,11 +1166,17 @@ def share_replayssm_ring_trackers(
ring_start = torch.zeros(num_blocks, dtype=torch.int32, device=device)
prev_num_accepted = torch.zeros_like(ring_start)
prev_query_len = torch.zeros_like(ring_start)
for layer_name in group_layer_names:
mixer = replayssm_mixers[layer_name]
mixer._replayssm_ring_start = ring_start
mixer._replayssm_prev_num_accepted = prev_num_accepted
mixer._updates_replayssm_trackers = layer_name == last_layer_name
mixer._replayssm_prev_query_len = prev_query_len
mixer._commits_replayssm_trackers = False
mixer._updates_replayssm_trackers = False
replayssm_mixers[first_layer_name]._commits_replayssm_trackers = True
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
def mamba_mixer2(
@@ -227,6 +227,7 @@ class MambaStateShapeCalculator:
tp_world_size: int,
logical_window: int,
backend: MambaBackendEnum,
num_speculative_tokens: int = 0,
) -> tuple[
tuple[int, int],
tuple[int, int, int],
@@ -241,8 +242,8 @@ class MambaStateShapeCalculator:
"""
ring_buffer_len = logical_window
if backend == MambaBackendEnum.FLASHINFER:
# FlashInfer keeps the live window and appended token together.
ring_buffer_len += 1
# FlashInfer keeps the live window and current verify window together.
ring_buffer_len += 1 + num_speculative_tokens
local_nheads, head_dim, state_size = base_shapes[1]
local_ngroups = divide(n_groups, tp_world_size)
return (
@@ -14,7 +14,6 @@ from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
)
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_cpu,
causal_conv1d_update_torch,
)
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import (
@@ -56,7 +55,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices: torch.Tensor,
has_initial_state: torch.Tensor,
silu_activation: bool,
is_vnni: bool,
is_weight_packed: bool,
) -> torch.Tensor:
if not is_conv_state_dim_first():
return ops.causal_conv1d_fwd_cpu(
@@ -68,7 +67,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=silu_activation,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
scratch, identity_indices = _stage_ds_conv_states(conv_states, cache_indices)
@@ -81,7 +80,7 @@ def _causal_conv1d_fwd_cpu(
cache_indices=identity_indices,
has_initial_state=has_initial_state,
silu_activation=silu_activation,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
_scatter_ds_conv_states(conv_states, cache_indices, scratch)
return out
@@ -95,7 +94,7 @@ def _causal_conv1d_update_cpu(
bias: torch.Tensor | None,
silu_activation: bool,
conv_state_indices: torch.Tensor,
is_vnni: bool,
is_weight_packed: bool,
num_accepted_tokens: torch.Tensor | None = None,
) -> torch.Tensor:
if not is_conv_state_dim_first():
@@ -106,7 +105,7 @@ def _causal_conv1d_update_cpu(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
@@ -118,13 +117,20 @@ def _causal_conv1d_update_cpu(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=identity_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
_scatter_ds_conv_states(conv_states, conv_state_indices, scratch)
return out
def is_arm_bf16() -> bool:
return (
current_platform.get_cpu_architecture() == CpuArchEnum.ARM
and torch.cpu.get_capabilities().get("bf16", False)
)
def cpu_gdn_attention_core(
mixed_qkv: torch.Tensor,
b: torch.Tensor,
@@ -201,9 +207,10 @@ def _cpu_gdn_attention_nonspec(
assert state_indices_tensor is not None
assert query_start_loc is not None
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
use_cpp_conv = torch.cpu._is_avx512_bf16_supported()
# C++ conv (conv.cpp) runs on aarch64 with bf16 support and on x86 uses VDPBF16PS,
# not AMX tiles, so it runs on any AVX-512BF16 CPU. The weights are packed on this
# same predicate at load time.
use_cpp_conv = torch.cpu._is_avx512_bf16_supported() or is_arm_bf16()
conv_state = layer.kv_cache[0]
if use_cpp_conv:
@@ -244,28 +251,17 @@ def _cpu_gdn_attention_nonspec(
bias=layer.conv1d.bias,
silu_activation=(layer.activation == "silu"),
conv_state_indices=decode_state_indices,
is_vnni=True,
is_weight_packed=True,
)
else:
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
decode_conv_state = conv_state[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
@@ -315,7 +311,7 @@ def _cpu_gdn_attention_nonspec(
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
prefill_mixed_qkv = causal_conv1d_torch(
@@ -383,9 +379,9 @@ def _ssm_state_view(layer) -> torch.Tensor:
def _unpacked_conv_weight(layer) -> torch.Tensor:
"""Return the plain (dim, width) conv weight.
On AMX the conv1d weight is VNNI-packed in place at load time (only usable
by the AMX C++ kernel), so the torch spec-decode path relies on the
un-packed copy stashed by ``dispatch_cpu_unquantized_gemm``.
On AMX and aarch64+bf16 the conv1d weight is packed in place at load time,
so the torch spec-decode path relies on the un-packed copy stashed by
``dispatch_cpu_unquantized_gemm``.
"""
w = getattr(layer.conv1d, "_cpu_unpacked_conv_weight", None)
if w is not None:
@@ -505,13 +501,15 @@ def _spec_forward(
seq_starts = spec_qsl_cpu[:-1]
seq_lens = spec_qsl_cpu[1:] - spec_qsl_cpu[:-1]
is_amx = torch.cpu._is_amx_tile_supported()
# ---- 1. Convolution (per-sequence rolling buffer) ----
dim = mixed_qkv_spec.size(-1)
bias = layer.conv1d.bias
silu = layer.activation == "silu"
can_use_native_conv = (
torch.cpu._is_amx_tile_supported()
(is_amx or is_arm_bf16())
and width == 4
and num_spec_decodes > 0
and bool(torch.all(seq_lens == seq_lens[0]).item())
@@ -528,7 +526,7 @@ def _spec_forward(
conv_state_indices=spec_state_indices[:num_spec_decodes, 0]
.to("cpu", torch.int32)
.contiguous(),
is_vnni=True,
is_weight_packed=True,
num_accepted_tokens=num_accepted[:num_spec_decodes].to("cpu", torch.int32),
).view_as(mixed_qkv_spec)
else:
@@ -603,9 +601,9 @@ def _spec_aware_nonspec(
assert query_start_loc is not None
state_indices_tensor = state_indices_tensor.contiguous()
is_amx = torch.cpu._is_amx_tile_supported()
can_use_native_conv = torch.cpu._is_amx_tile_supported() or is_arm_bf16()
if not is_amx:
if not can_use_native_conv:
conv_weights = _unpacked_conv_weight(layer)
num_decodes = attn_metadata_i.num_decodes
@@ -618,7 +616,8 @@ def _spec_aware_nonspec(
decode_b = b[:num_decode_tokens]
decode_a = a[:num_decode_tokens]
decode_state_indices = state_indices_tensor[:num_decodes]
if is_amx:
if can_use_native_conv:
decode_mixed_qkv = _causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_states=conv_buf,
@@ -626,30 +625,19 @@ def _spec_aware_nonspec(
bias=layer.conv1d.bias,
silu_activation=layer.activation == "silu",
conv_state_indices=decode_state_indices,
is_vnni=True,
is_weight_packed=True,
)
else:
# Only the first ``width-1`` columns hold the real conv state.
conv_state_view = conv_buf[:, :, : width - 1]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state_view[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state_view,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state_view,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu(
@@ -685,7 +673,8 @@ def _spec_aware_nonspec(
prefill_has_initial_state = has_initial_state[
num_decodes : num_decodes + num_prefills
]
if is_amx:
if can_use_native_conv:
prefill_mixed_qkv = _causal_conv1d_fwd_cpu(
x=prefill_mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
@@ -695,7 +684,7 @@ def _spec_aware_nonspec(
cache_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
prefill_mixed_qkv = causal_conv1d_torch(
@@ -757,8 +746,9 @@ def _spec_aware_nonspec_subset(
prefill_state_indices = prefill_state_indices.contiguous()
is_amx = torch.cpu._is_amx_tile_supported()
can_use_native_conv = is_amx or is_arm_bf16()
if is_amx:
if can_use_native_conv:
conv_out = _causal_conv1d_fwd_cpu(
x=mixed_qkv.transpose(0, 1),
weight=layer.conv1d.weight,
@@ -768,7 +758,7 @@ def _spec_aware_nonspec_subset(
cache_indices=prefill_state_indices,
has_initial_state=has_initial_state,
silu_activation=layer.activation == "silu",
is_vnni=True,
is_weight_packed=True,
).transpose(0, 1)
else:
conv_weights = _unpacked_conv_weight(layer)
@@ -31,6 +31,7 @@ logger = init_logger(__name__)
def _update_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
state_batch_indices_stride,
n_slots,
@@ -52,6 +53,9 @@ def _update_replayssm_ring_trackers_kernel(
if RESET:
tl.store(ring_start + slots, 0, mask=valid)
tl.store(prev_num_accepted + slots, 0, mask=valid)
# MTP alone reads prev_query_len. Standard decode deliberately leaves
# it unchanged; reset clears the pending MTP window after prefill.
tl.store(prev_query_len + slots, 0, mask=valid)
else:
prev = tl.load(prev_num_accepted + slots, mask=valid, other=0)
start = tl.load(ring_start + slots, mask=valid, other=0)
@@ -66,9 +70,70 @@ def _update_replayssm_ring_trackers_kernel(
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
@triton.jit(
do_not_specialize=["n_slots", "state_batch_indices_stride"],
do_not_specialize_on_alignment=["state_batch_indices"],
)
def _commit_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
num_accepted_tokens,
query_start_loc,
state_batch_indices_stride,
n_slots,
num_states,
logical_window: tl.constexpr,
ring_buffer_len: tl.constexpr,
pad_slot_id: tl.constexpr,
BLOCK: tl.constexpr,
) -> None:
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < n_slots
slots = tl.load(
state_batch_indices + offsets * state_batch_indices_stride,
mask=mask,
other=pad_slot_id,
)
valid = mask & (slots != pad_slot_id) & (slots >= 0) & (slots < num_states)
accepted_since_checkpoint = tl.load(prev_num_accepted + slots, mask=valid, other=0)
current_ring_start = tl.load(ring_start + slots, mask=valid, other=0)
previous_speculative_query_len = tl.load(
prev_query_len + slots, mask=valid, other=0
)
accepted_from_previous_query = tl.load(
num_accepted_tokens + offsets, mask=mask, other=0
)
must_checkpoint = (previous_speculative_query_len > 0) & (
accepted_since_checkpoint + previous_speculative_query_len > logical_window
)
next_start = tl.where(
must_checkpoint,
(current_ring_start + accepted_since_checkpoint) % ring_buffer_len,
current_ring_start,
)
next_prev = tl.where(
previous_speculative_query_len == 0,
0,
tl.where(
must_checkpoint,
accepted_from_previous_query,
accepted_since_checkpoint + accepted_from_previous_query,
),
)
current_speculative_query_len = tl.load(
query_start_loc + offsets + 1, mask=mask, other=0
) - tl.load(query_start_loc + offsets, mask=mask, other=0)
tl.store(ring_start + slots, next_start, mask=valid)
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
tl.store(prev_query_len + slots, current_speculative_query_len, mask=valid)
def update_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
logical_window: int | None = None,
ring_buffer_len: int | None = None,
@@ -90,10 +155,15 @@ def update_replayssm_ring_trackers(
_update_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
state_batch_indices.stride(0),
n_slots,
min(ring_start.numel(), prev_num_accepted.numel()),
min(
ring_start.numel(),
prev_num_accepted.numel(),
prev_query_len.numel(),
),
logical_window,
ring_buffer_len,
pad_slot_id,
@@ -105,6 +175,7 @@ def update_replayssm_ring_trackers(
def reset_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
@@ -112,11 +183,62 @@ def reset_replayssm_ring_trackers(
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
pad_slot_id=pad_slot_id,
)
def commit_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
num_accepted_tokens: torch.Tensor,
query_start_loc: torch.Tensor,
logical_window: int,
ring_buffer_len: int,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
"""Commit the preceding speculative window and record the current one.
MTP evaluates a target token and its draft tokens together, but the number
accepted from that query is available only on the next forward pass. This
function then advances the per-request ring by the accepted prefix and
records the current query length for the following pass. A zero
``prev_query_len`` means that reset/prefill left no prior MTP query to
commit; slot validity is handled independently by the kernel mask.
Standard single-token decode needs no delayed commit: its ReplaySSM kernel
advances the shared ring trackers directly after every token.
"""
if state_batch_indices.dim() > 1:
state_batch_indices = state_batch_indices[:, 0]
n_slots = state_batch_indices.numel()
if n_slots == 0:
return
block = 128
_commit_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
num_accepted_tokens,
query_start_loc,
state_batch_indices.stride(0),
n_slots,
min(
ring_start.numel(),
prev_num_accepted.numel(),
prev_query_len.numel(),
),
logical_window,
ring_buffer_len,
pad_slot_id,
BLOCK=block,
)
class MambaSSUBackend(ABC):
"""Abstract base class for Mamba SSU backends."""
@@ -385,6 +507,7 @@ def selective_state_update_replayssm_flashinfer(
dt_cache: torch.Tensor,
ring_start: torch.Tensor,
prev_num_accepted_tokens: torch.Tensor,
prev_query_len: torch.Tensor,
logical_window: int,
D: torch.Tensor | None = None,
dt_bias: torch.Tensor | None = None,
@@ -395,6 +518,9 @@ def selective_state_update_replayssm_flashinfer(
update_trackers: bool = True,
enable_stochastic_rounding: bool = False,
stochastic_rounding_philox_rounds: int = 0,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: int | None = None,
enable_pdl: bool = False,
) -> torch.Tensor:
"""Run FlashInfer checkpointing SSU and optionally advance shared trackers."""
if _flashinfer_replayssm_kernel is None:
@@ -404,11 +530,12 @@ def selective_state_update_replayssm_flashinfer(
)
if x.dim() == 3:
x = x.unsqueeze(1)
dt = dt.unsqueeze(1)
B = B.unsqueeze(1)
C = C.unsqueeze(1)
out = out.unsqueeze(1)
dim = 0 if cu_seqlens is not None else 1
x = x.unsqueeze(dim)
dt = dt.unsqueeze(dim)
B = B.unsqueeze(dim)
C = C.unsqueeze(dim)
out = out.unsqueeze(dim)
indices = state_batch_indices
if indices is not None and indices.dim() > 1:
@@ -443,6 +570,9 @@ def selective_state_update_replayssm_flashinfer(
pad_slot_id=null_block_id,
rand_seed=rand_seed,
philox_rounds=stochastic_rounding_philox_rounds or 10,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
enable_pdl=enable_pdl,
cb_scaled=cb_scaled,
cumAdt_vec=cumAdt_vec,
cb_old=cb_old,
@@ -451,6 +581,7 @@ def selective_state_update_replayssm_flashinfer(
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted_tokens,
prev_query_len,
indices,
logical_window=logical_window,
ring_buffer_len=x_cache.size(2),
@@ -1208,9 +1208,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
"""Override the default vLLM placeholder scales with the llm-compressor loaded
scales. Zero points are not used as only symmetric quantization is supported.
"""
layer._k_scale = layer.k_scale
layer._v_scale = layer.v_scale
layer._q_scale = layer.q_scale
# Buffers (not Parameters) so sleep snapshots them; ATTN_HEAD changes shape.
layer.register_buffer("_k_scale", layer.k_scale.detach())
layer.register_buffer("_v_scale", layer.v_scale.detach())
layer.register_buffer("_q_scale", layer.q_scale.detach())
# Set the _float variants that the attention backend uses.
def _to_scalar(tensor: torch.Tensor) -> float:
@@ -210,6 +210,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
# Per-expert biases, as gpt-oss carries.
if self.moe.has_bias:
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
w13_num_shards * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
requires_grad=False,
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
self.group_size = -1
@@ -350,7 +371,41 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
routing_tables=layer._expert_routing_tables(),
)
if self.wna16_backend == WNA16MoEBackend.ZEN_CPU:
# Monolithic experts route inside apply(), so like the unquantized
# CPU path they need the router config that only the layer carries.
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
swigluoai_w13_interleave_perm,
)
# Reorder before the conversion below, which repacks w13 for the kernel.
# The CT WNA16 buffers are N-first, so 2I is dim 1 of the packed weight
# [E, 2I, K/pack] and of its group scales [E, 2I, G], as for the bias.
perm = swigluoai_w13_interleave_perm(
self.experts_cls,
self.moe.activation,
layer.w13_weight_packed.size(1),
layer.w13_weight_packed.device,
)
if perm is not None:
replace_parameter(
layer,
"w13_weight_packed",
layer.w13_weight_packed.data[:, perm].contiguous(),
)
replace_parameter(
layer,
"w13_weight_scale",
layer.w13_weight_scale.data[:, perm].contiguous(),
)
if self.moe.has_bias:
replace_parameter(
layer, "w13_bias", layer.w13_bias.data[:, perm].contiguous()
)
# Process weights using the shared oracle infrastructure
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
@@ -363,6 +418,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_scale=layer.w2_weight_scale,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
if converted is None:
@@ -378,8 +435,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
w13_bias,
w2_bias,
) = converted
# Replace common parameters
@@ -388,6 +445,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
if w13_bias is not None:
replace_parameter(layer, "w13_bias", w13_bias)
if w2_bias is not None:
replace_parameter(layer, "w2_bias", w2_bias)
# CPU fused_experts_cpu requires zero points even for symmetric quant.
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
if (
@@ -437,6 +499,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
num_bits=self.num_bits,
w1_zp=getattr(layer, "w13_weight_zero_point", None),
w2_zp=getattr(layer, "w2_weight_zero_point", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
@@ -736,6 +736,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w2_bias=w2_bias,
_cache_permute_indices=self._cache_permute_indices,
activation=self.moe.activation,
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
)
)
@@ -220,6 +220,7 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
w2_weight_scale=w2_scale,
w13_bias=w13_bias,
w2_bias=w2_bias,
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
)
)
@@ -9,8 +9,8 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_hidden_dim_for_fi,
align_fp4_moe_weights_for_fi,
align_trtllm_fp4_moe_hidden_dim_for_fi,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
swizzle_blockscale,
@@ -128,9 +128,9 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
]:
"""Prepare weights for the CuteDSL wrapper-based NvFP4 MoE backend.
Converts weight scale factors to MMA layout expected by CuteDslMoEWrapper,
and interleaves w13 gate/linear rows for gated activations. Non-gated
activations use a single w13 projection and keep its row order unchanged.
Pads the runtime expert tensors to the kernel's GEMM alignment, converts
weight scale factors to the MMA layout expected by CuteDslMoEWrapper, and
interleaves w13 gate/linear rows for gated activations.
"""
# Global scaling factors (same as other FlashInfer backends).
num_experts = w13.shape[0]
@@ -140,12 +140,38 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
)
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(num_experts)
if layer.activation.is_gated:
gated = layer.activation.is_gated
if gated:
w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
layer.activation, w13, w13_scale
)
# Interleave up/gate rows for w13 weights and scales.
# GEMM1's output dimension must be a multiple of 128: 2I for gated
# activations (also required by interleaving), but only I for non-gated.
# Keep the checkpoint tensors unchanged and pad only the kernel's runtime
# representation. Zero rows also make the padded GEMM2 contraction a no-op.
w13, w13_scale, w2, w2_scale, padded_intermediate = align_fp4_moe_weights_for_fi(
w13,
w13_scale,
w2,
w2_scale,
is_act_and_mul=gated,
min_alignment=64 if gated else 128,
)
layer.moe_config.intermediate_size_per_partition = padded_intermediate
# GEMM1 gathers full 256-element K tiles of activations and their scales
# without a K-tail predicate. Pad runtime weights to match; the MoE runner
# pads activations and trims the output using the original hidden size.
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
layer.moe_config.hidden_dim = padded_hidden
if gated:
# Interleave up/gate rows for the fused gated activation.
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
@@ -371,8 +397,8 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
w13, w13_scale, w2, w2_scale, padded_hidden = (
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
@@ -13,6 +13,8 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
def activation_to_flashinfer_int(activation: MoEActivation) -> int:
return activation_to_flashinfer_type(activation).value
@@ -50,6 +52,20 @@ def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType"
return ACTIVATION_TO_FI_ACTIVATION[activation]
def quantize_nvfp4_per_token_input(
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Quantize NVFP4 activations with one FP32 decode scale per token."""
from flashinfer import SfLayout, nvfp4_quantize
return nvfp4_quantize(
hidden_states,
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
)
def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
return (
x.reshape(-1, 2, x.shape[-2] // 2, x.shape[-1]).flip(dims=[1]).reshape(x.shape)
@@ -264,7 +280,7 @@ def align_fp4_moe_weights_for_fi(
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
def align_trtllm_fp4_moe_hidden_dim_for_fi(
def align_fp4_moe_hidden_dim_for_fi(
w13: torch.Tensor,
w13_scale: torch.Tensor,
w2: torch.Tensor,
@@ -279,7 +295,7 @@ def align_trtllm_fp4_moe_hidden_dim_for_fi(
return w13, w13_scale, w2, w2_scale, hidden_size
logger.warning_once(
"Padding hidden size from %d to %d for TRTLLM NVFP4 MoE weights. "
"Padding hidden size from %d to %d for FlashInfer NVFP4 MoE weights. "
"This requires activation slicing at runtime and may cause "
"performance degradation.",
hidden_size,
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse attention indexer layers on aiter's paged MXFP4 kernels, for ROCm.
With ``indexer_kv_dtype="mxfp4"`` on gfx950, the model writes the indexer K
cache in the order aiter's MQA-logits kernel reads (see
`rocm_paged_mxfp4_cache_layout`), and the kernel scores the cache in place.
Requires `RocmMxfp4IndexerMetadataBuilder` metadata.
"""
import torch
from torch import nn
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import (
rocm_mxfp4_sparse_attn_indexer,
rocm_mxfp4_sparse_mqa_indexer,
)
class RocmSparseAttnIndexer(SparseAttnIndexer):
"""`SparseAttnIndexer` whose HIP path walks the paged MXFP4 cache with
aiter's kernel. A two-level indexer's source layer also publishes its
candidate pool here."""
def forward_hip(
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
):
assert isinstance(q_quant, tuple) and self.skip_k_cache_insert, (
"the ROCm MXFP4 indexer takes (values, scales) Q and a K cache "
"the model already wrote"
)
q_values, q_scale = q_quant
return rocm_mxfp4_sparse_attn_indexer(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_values,
q_scale,
weights,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.topk_indices_buffer,
compress_ratio=self.compress_ratio,
candidate_blocks=self.candidate_blocks,
candidate_block_size=self.candidate_block_size,
candidate_write=self.candidate_write,
)
class RocmSparseMQAIndexer(nn.Module):
"""DeepSeek-V4.1's candidate-consuming indexer on aiter's paged MXFP4
MQA-logits kernel: the consumers score only the candidate pool the source
layer published, read straight from the paged cache.
Built and called as `SparseMQAIndexer` is, so the model can take either.
"""
weights_dtype = torch.float32
"""The scorer's weights and logits stay fp32: gfx950 has no packed bf16
arithmetic, so its head reduce costs the same in either dtype."""
def __init__(
self,
k_cache,
topk_tokens: int,
head_dim: int,
max_total_seq_len: int,
topk_indices_buffer: torch.Tensor,
candidate_blocks: torch.Tensor,
candidate_block_size: int,
):
super().__init__()
self.k_cache = k_cache
self.topk_tokens = topk_tokens
self.head_dim = head_dim
# aiter's kernel bounds a row by the model length in compressed
# positions, not by the prefill buffer SparseMQAIndexer sizes.
model_config = get_current_vllm_config().model_config
self.max_model_len = model_config.max_model_len // k_cache.compress_ratio
self.topk_indices_buffer = topk_indices_buffer
self.candidate_blocks = candidate_blocks
self.candidate_block_size = candidate_block_size
self.num_candidate_cols = candidate_blocks.shape[1] * candidate_block_size
def forward(
self,
hidden_states: torch.Tensor,
q_quant: tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
) -> torch.Tensor:
assert k is None, "the model writes the indexer K cache"
q_values, q_scale = q_quant
return rocm_mxfp4_sparse_mqa_indexer(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_values,
q_scale,
weights,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.topk_indices_buffer,
self.k_cache.compress_ratio,
self.candidate_blocks,
self.candidate_block_size,
self.num_candidate_cols,
)
@@ -335,6 +335,7 @@ def sparse_attn_indexer(
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_pcp: bool,
pcp_shard_decode_requests: bool,
dense_mha_metadata_layer_name: LayerNameType,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
@@ -404,6 +405,7 @@ def sparse_attn_indexer(
topk_indices_buffer,
skip_k_cache_insert,
use_pcp,
pcp_shard_decode_requests,
dense_mha_metadata_layer_name,
use_fp4_cache,
candidate_blocks=candidate_blocks,
@@ -442,6 +444,7 @@ def sparse_attn_indexer(
slot_mapping,
num_decode_tokens,
use_pcp,
pcp_shard_decode_requests=pcp_shard_decode_requests,
)
# scale_fmt can be None, but the function expects str
assert scale_fmt is not None
@@ -817,6 +820,7 @@ def sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_pcp: bool,
pcp_shard_decode_requests: bool,
dense_mha_metadata_layer_name: LayerNameType,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
@@ -898,6 +902,7 @@ class SparseAttnIndexer(CustomOp):
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
self._cp_kv_cache_interleave_size: int | None = None
if current_platform.is_cuda() and not has_deep_gemm():
raise RuntimeError(
@@ -994,6 +999,7 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer,
self.skip_k_cache_insert,
self.use_pcp,
self.pcp_shard_decode_requests,
_encode_layer_name(self.dense_mha_metadata_layer_name),
self.use_fp4_cache,
self.dcp_rank,
+22 -8
View File
@@ -475,13 +475,27 @@ def dispatch_cpu_unquantized_gemm(
# Skip CPU GEMM dispatch for non-2D weights (e.g. MoE 3D expert weights).
# These layers are handled by their own specialized methods.
if layer.weight.ndim != 2:
# this is not a linear layer
# For now it should be a causal_conv1d op or MoE 3D expert weights
# The C++ causal_conv1d kernels use VDPBF16PS (no AMX tiles), so the
# VNNI weight prepack applies to any AVX-512BF16 CPU, not just AMX
# (e.g. AMD Zen5/Turin).
if torch.cpu._is_avx512_bf16_supported() and hasattr(
ops, "causal_conv1d_weight_pack"
# This is not a linear layer.
# For now it should be a causal_conv1d op or MoE 3D expert weights.
# Causal-conv weights use [dim, 1, width].
# The C++ causal_conv1d kernels support aarch64 with bf16 and
# x86 CPUs that use VDPBF16PS (no AMX tiles). So the
# weight prepack applies to aarch64 with bf16 and any AVX-512BF16 CPU,
# not just AMX (e.g. AMD Zen5/Turin).
is_causal_conv1d_weight = (
layer.weight.ndim == 3
and layer.weight.size(1) == 1
and layer.weight.size(2) == 4
and layer.weight.dtype == torch.bfloat16
)
is_arm_bf16 = (
current_platform.get_cpu_architecture() == CpuArchEnum.ARM
and torch.cpu.get_capabilities().get("bf16", False)
)
if (
(is_arm_bf16 or torch.cpu._is_avx512_bf16_supported())
and is_causal_conv1d_weight
and hasattr(ops, "causal_conv1d_weight_pack")
):
# prepack conv weight
unpacked = (
@@ -493,7 +507,7 @@ def dispatch_cpu_unquantized_gemm(
.clone()
)
# Stash the un-packed (dim, width) weight so the speculative-decode
# GDN path (which uses torch conv, not the C++ kernel) can use it.
# and unsupported-layout fallbacks can still use ordinary weights.
layer._cpu_unpacked_conv_weight = unpacked
layer.weight.data = ops.causal_conv1d_weight_pack(unpacked)
return
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from abc import ABC, abstractmethod
import torch
@@ -28,6 +29,11 @@ class BaseModelLoader(ABC):
def __init__(self, load_config: LoadConfig):
self.load_config = load_config
def get_external_weight_memory(self, vllm_config: VllmConfig) -> int:
"""Get weights memory from external process;
0 when the weights are not external."""
return 0
@abstractmethod
def download_model(self, model_config: ModelConfig) -> None:
"""Download a model so that it can be immediately loaded."""
@@ -65,11 +71,17 @@ class BaseModelLoader(ABC):
target_device = torch.device(load_device)
with set_default_torch_dtype(model_config.dtype):
with target_device:
time_before_load = time.perf_counter()
model = self.create_model(
vllm_config=vllm_config,
model_config=model_config,
prefix=prefix,
)
time_after_load = time.perf_counter()
logger.info_once(
"Initializing model took %.6f seconds",
time_after_load - time_before_load,
)
logger.debug("Loading weights on %s ...", load_device)
self.load_weights(model, model_config)
@@ -17,8 +17,7 @@ Engines then load from the daemons with:
vllm serve /path/to/model --tensor-parallel-size 4 \\
--load-format ipc_cache
Tensor, expert and data parallelism are supported; pipeline parallelism is
rejected at launch.
Tensor, expert, data and pipeline parallelism are supported.
For multi-node tensor parallelism, run one launcher per node with a shared
rendezvous so the global TP group forms across nodes (CUDA IPC handles are
@@ -40,7 +39,8 @@ node-local, so each node serves only its local GPUs' shards). Reuse the same
The global TP rank of local GPU ``i`` on node ``r`` is
``r * (tp_size // nnodes) + i``, matching vLLM's contiguous per-node rank
assignment, so each engine worker maps its shard from the daemon on its own
node.
node. With pipeline parallelism each node additionally runs one daemon per PP
stage; the global rank is ``pp_rank * tp_size + tp_rank``.
For data parallelism (e.g. a TP1 x DP16 x EP decode fleet) run one launcher per
node with the engine's DP placement flags. Local GPU ``i`` serves DP rank
@@ -78,6 +78,7 @@ cached and keep loading from disk in the engine.
import contextlib
import fcntl
import gc
import multiprocessing
import os
import socket
@@ -116,6 +117,7 @@ from vllm.model_executor.model_loader.weight_cache.utils import (
is_draft_model_cacheable,
)
from vllm.platforms import current_platform
from vllm.utils.mem_utils import format_gib
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.worker.workspace import init_workspace_manager
@@ -168,27 +170,30 @@ def export_entries(
class WeightCacheDaemon:
"""Per-GPU process that loads one TP shard and serves CUDA IPC handles."""
"""Per-GPU process that loads one TP/PP shard and serves CUDA IPC handles."""
def __init__(
self,
vllm_config: VllmConfig,
tp_rank: int,
global_rank: int,
local_rank: int,
distributed_init_method: str,
socket_dir: str | None = None,
is_draft: bool = False,
dp_rank: int = 0,
pp_rank: int = 0,
):
parallel_config = vllm_config.parallel_config
if parallel_config.data_parallel_size > 1:
self.tp_size = parallel_config.tensor_parallel_size
self.pp_size = parallel_config.pipeline_parallel_size
self.dp_size = parallel_config.data_parallel_size
if self.dp_size > 1:
# The engine sets these on each DP rank's workers; the MoE/EP
# layers read them to place their expert shards.
tp_size = parallel_config.tensor_parallel_size
if parallel_config.nnodes > 1:
world_size = parallel_config.data_parallel_size * tp_size
world_size = self.dp_size * self.tp_size
local_world = world_size // parallel_config.nnodes
start_dp_rank = parallel_config.node_rank * local_world // tp_size
start_dp_rank = parallel_config.node_rank * local_world // self.tp_size
else:
start_dp_rank = parallel_config.data_parallel_rank
parallel_config = replace(
@@ -206,12 +211,12 @@ class WeightCacheDaemon:
self.model_config = spec.draft_model_config
else:
self.model_config = vllm_config.model_config
self.tp_rank = tp_rank
self.global_rank = global_rank
rank_in_dp = global_rank - dp_rank * self.pp_size * self.tp_size
self.tp_rank = rank_in_dp % self.tp_size
self.pp_rank = rank_in_dp // self.tp_size
self.dp_rank = dp_rank
self.tp_size = parallel_config.tensor_parallel_size
self.dp_size = parallel_config.data_parallel_size
self.global_rank = dp_rank * self.tp_size + tp_rank
self.world_size = self.dp_size * self.tp_size
self.world_size = self.dp_size * self.pp_size * self.tp_size
self.local_rank = local_rank
self.distributed_init_method = distributed_init_method
self.socket_dir = socket_dir
@@ -223,7 +228,9 @@ class WeightCacheDaemon:
self.cache_config = WeightCacheKey.from_model_config(
self.model_config,
tp_size=self.tp_size,
tp_rank=tp_rank,
tp_rank=self.tp_rank,
pp_size=self.pp_size,
pp_rank=self.pp_rank,
dp_size=self.dp_size,
dp_rank=dp_rank,
is_draft=is_draft,
@@ -242,12 +249,19 @@ class WeightCacheDaemon:
backend=current_platform.dist_backend,
)
with set_current_vllm_config(self.vllm_config):
ensure_model_parallel_initialized(self.tp_size, 1)
ensure_model_parallel_initialized(self.tp_size, self.pp_size)
self.model = self.get_model()
# Loading and post-processing leave freed transients in the caching
# allocator; return them so engines sharing the GPU can use them.
gc.collect()
torch.accelerator.empty_cache()
logger.info(
"Weight cache %s daemon rank %d loaded model",
"Weight cache %s daemon rank %d loaded model (%s GiB allocated, "
"%s GiB reserved)",
self.role,
self.global_rank,
format_gib(torch.accelerator.memory_allocated()),
format_gib(torch.accelerator.memory_reserved()),
)
def get_model(self) -> torch.nn.Module:
@@ -364,6 +378,14 @@ class WeightCacheDaemon:
cmd = request.get("cmd")
if cmd == "get_state":
self._handle_get_state(conn, request)
elif cmd == "get_memory":
send_msg(
conn,
{
"status": "ok",
"memory_bytes": torch.accelerator.memory_allocated(),
},
)
elif cmd == "release":
self._handle_release(conn)
else:
@@ -414,7 +436,7 @@ class WeightCacheDaemon:
def _run_daemon(
tp_rank: int,
global_rank: int,
local_rank: int,
vllm_config: VllmConfig,
distributed_init_method: str,
@@ -422,15 +444,17 @@ def _run_daemon(
ready_queue: "multiprocessing.Queue[tuple[str, int]]",
is_draft: bool = False,
dp_rank: int = 0,
pp_rank: int = 0,
) -> None:
daemon = WeightCacheDaemon(
vllm_config,
tp_rank,
global_rank,
local_rank,
distributed_init_method,
socket_dir,
is_draft,
dp_rank,
pp_rank,
)
daemon.load_model()
daemon.serve_forever(
@@ -438,26 +462,34 @@ def _run_daemon(
)
def plan_local_ranks(parallel_config: ParallelConfig) -> list[tuple[int, int, int]]:
"""``(local_rank, dp_rank, tp_rank)`` for every GPU this launcher serves.
def plan_local_ranks(
parallel_config: ParallelConfig,
) -> list[tuple[int, int, int, int]]:
"""``(local_rank, dp_rank, pp_rank, tp_rank)`` for every local daemon.
Global ranks enumerate DP then TP: ``global = dp_rank * tp_size + tp_rank``.
Global ranks enumerate DP, PP, then TP: ``global = dp_rank * pp_size *
tp_size + pp_rank * tp_size + tp_rank``.
With ``--nnodes`` the engine hands each node a contiguous block of global
ranks, so node ``r`` serves ``node_rank * local + i``; without it a
launcher's block starts at ``--data-parallel-start-rank * tp_size``.
"""
tp_size = parallel_config.tensor_parallel_size
world_size = parallel_config.data_parallel_size * tp_size
pp_size = parallel_config.pipeline_parallel_size
world_size = parallel_config.data_parallel_size * pp_size * tp_size
if parallel_config.nnodes > 1:
local_world_size = world_size // parallel_config.nnodes
base = parallel_config.node_rank * local_world_size
else:
local_world_size = parallel_config.data_parallel_size_local * tp_size
base = parallel_config.data_parallel_rank * tp_size
return [
(i, (base + i) // tp_size, (base + i) % tp_size)
for i in range(local_world_size)
]
local_world_size = parallel_config.data_parallel_size_local * pp_size * tp_size
base = parallel_config.data_parallel_rank * pp_size * tp_size
placements = []
for local_rank in range(local_world_size):
global_rank = base + local_rank
rank_in_dp = global_rank % (pp_size * tp_size)
dp_rank = global_rank // (pp_size * tp_size)
pp_rank, tp_rank = divmod(rank_in_dp, tp_size)
placements.append((local_rank, dp_rank, pp_rank, tp_rank))
return placements
def get_draft_daemon_config(vllm_config: VllmConfig) -> VllmConfig | None:
@@ -469,21 +501,18 @@ def get_draft_daemon_config(vllm_config: VllmConfig) -> VllmConfig | None:
def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
"""Reject pipeline parallelism and placements the daemon cannot map."""
if parallel_config.pipeline_parallel_size > 1:
raise ValueError(
"The weight cache daemon only supports tensor, expert and data "
"parallelism; pipeline parallelism is not supported"
)
"""Reject placements the daemon cannot map."""
dp_size = parallel_config.data_parallel_size
tp_size = parallel_config.tensor_parallel_size
pp_size = parallel_config.pipeline_parallel_size
if parallel_config.nnodes > 1:
world_size = dp_size * tp_size
world_size = dp_size * pp_size * tp_size
if world_size % parallel_config.nnodes != 0:
raise ValueError(
f"--nnodes ({parallel_config.nnodes}) must evenly divide the "
f"daemon world size ({world_size} = data-parallel-size "
f"{dp_size} x tensor-parallel-size {tp_size})"
f"{dp_size} x pipeline-parallel-size {pp_size} x "
f"tensor-parallel-size {tp_size})"
)
return
if dp_size == 1:
@@ -6,7 +6,9 @@ daemon via CUDA IPC instead of loading from disk."""
import dataclasses
import socket
import time
from collections.abc import Callable
from copy import copy
from typing import TypeVar
import torch
import torch.nn as nn
@@ -15,6 +17,7 @@ from vllm.config import ModelConfig, VllmConfig
from vllm.config.load import LoadConfig
from vllm.distributed import (
get_dp_group,
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
@@ -22,6 +25,7 @@ from vllm.logger import init_logger
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.model_executor.model_loader.utils import (
get_draft_load_config,
initialize_model,
process_weights_after_loading,
)
@@ -39,6 +43,9 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
send_msg,
verify_socket_owner,
)
from vllm.model_executor.model_loader.weight_cache.utils import (
is_draft_model_cacheable,
)
from vllm.model_executor.utils import weights_already_processed
from vllm.tracing import instrument
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -49,6 +56,8 @@ _CONNECT_TIMEOUT_S = 5.0
_STATE_TIMEOUT_S = 300.0
_STARTUP_RETRY_INTERVAL_S = 0.5
_T = TypeVar("_T")
class IpcModelLoader(BaseModelLoader):
"""Loads a model by mapping the weight cache daemon's tensors via CUDA IPC.
@@ -109,6 +118,52 @@ class IpcModelLoader(BaseModelLoader):
f"{load_config.load_format}: {sorted(extra_config)}"
)
def get_external_weight_memory(self, vllm_config: VllmConfig) -> int:
# Copy mode clones the weights into this process; nothing external.
if self.mode != "zero_copy":
return 0
total = self._daemon_memory()
if is_draft_model_cacheable(vllm_config.speculative_config):
# The draft group is queried with the same config the engine
# would load the draft with.
draft_config = get_draft_load_config(vllm_config)
if draft_config.load_format == "ipc_cache":
draft_loader = IpcModelLoader(draft_config)
# Only a zero-copy draft-group daemon holds external weights;
# an explicit draft_load_config without is_draft would resolve
# back to the target socket and double-count it.
if draft_loader.is_draft and draft_loader.mode == "zero_copy":
total += draft_loader._daemon_memory()
return total
def _daemon_memory(self) -> int:
if not self.fallback:
# The loader waits for the daemon at load time, so the weights
# will be zero-copy mapped for sure; mirror that wait here since
# returning 0 would over-grant the memory budget.
return self._with_startup_wait(self._query_daemon_memory)
# An unreachable daemon means a disk load, i.e. nothing external.
try:
return self._query_daemon_memory()
except (WeightCacheUnavailableError, ConnectionError, OSError) as e:
logger.warning(
"Cannot query weight cache daemon memory (%s); "
"assuming the weights are not externally held",
e,
)
return 0
def _query_daemon_memory(self) -> int:
with self._connect(self.connect_timeout_s) as conn:
send_msg(conn, {"cmd": "get_memory"})
response = recv_msg(conn)
if response.get("status") != "ok":
raise WeightCacheUnavailableError(
"Weight cache daemon rejected the memory query: "
f"{response.get('message')}"
)
return int(response.get("memory_bytes", 0))
def download_model(self, model_config: ModelConfig) -> None:
DefaultModelLoader(self._fallback_load_config()).download_model(model_config)
@@ -289,10 +344,13 @@ class IpcModelLoader(BaseModelLoader):
def _fetch_entries(self, model_config: ModelConfig) -> WeightCacheState:
dp_group = get_dp_group()
pp_group = get_pp_group()
cache_config = WeightCacheKey.from_model_config(
model_config,
tp_size=get_tensor_model_parallel_world_size(),
tp_rank=get_tensor_model_parallel_rank(),
pp_size=pp_group.world_size,
pp_rank=pp_group.rank_in_group,
dp_size=dp_group.world_size,
dp_rank=dp_group.rank_in_group,
is_draft=self.is_draft,
@@ -304,23 +362,25 @@ class IpcModelLoader(BaseModelLoader):
def _request_state_with_startup_wait(
self, cache_config: WeightCacheKey
) -> WeightCacheState:
return self._with_startup_wait(lambda: self._request_state(cache_config))
def _with_startup_wait(self, op: Callable[[], _T]) -> _T:
"""Retry op until the daemon answers or the state timeout elapses;
the daemon may still be loading the model when the engine starts."""
deadline = time.monotonic() + self.state_timeout_s
waiting_logged = False
while True:
try:
return self._request_state(cache_config)
except WeightCacheUnavailableError as e:
return op()
except (WeightCacheUnavailableError, ConnectionError, OSError) as e:
if time.monotonic() >= deadline:
raise WeightCacheUnavailableError(
"Weight cache daemon did not become ready within "
f"{self.state_timeout_s:.1f}s"
f"{self.state_timeout_s:.1f}s: {e}"
) from e
if not waiting_logged:
logger.info(
"Waiting up to %.1fs for the weight cache daemon to start",
self.state_timeout_s,
)
waiting_logged = True
logger.info_once(
"Waiting up to %.1fs for the weight cache daemon to start",
self.state_timeout_s,
)
time.sleep(
max(
0.0,
@@ -289,6 +289,8 @@ class WeightCacheKey:
quant_config_hash: str
revision: str | None
vllm_version: str
pp_size: int = 1
pp_rank: int = 0
is_draft: bool = False
"""Daemon group the weights come from; False is the target model."""
dp_size: int = 1
@@ -301,6 +303,8 @@ class WeightCacheKey:
tp_size: int,
tp_rank: int,
*,
pp_size: int = 1,
pp_rank: int = 0,
is_draft: bool = False,
dp_size: int = 1,
dp_rank: int = 0,
@@ -325,6 +329,8 @@ class WeightCacheKey:
model_arch=arch,
tp_size=tp_size,
tp_rank=tp_rank,
pp_size=pp_size,
pp_rank=pp_rank,
dtype=str(model_config.dtype),
quantization=model_config.quantization,
quant_config_hash=_hash_quant_config(quant_config),
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-parameter weight digests and reset, for verifying weight updates."""
import hashlib
from collections.abc import Iterator
import torch
import torch.nn as nn
from torch.utils._python_dispatch import is_traceable_wrapper_subclass
from vllm.model_executor.parameter import SharedWeightParameter
def _tensors(name: str, tensor: torch.Tensor) -> Iterator[tuple[str, torch.Tensor]]:
# Both keep their bytes in inner tensors (e.g. TorchAO's qdata and scale).
if isinstance(tensor, SharedWeightParameter):
for index, partition in tensor.partitions.items():
yield from _tensors(f"{name}.{index}", partition)
elif is_traceable_wrapper_subclass(tensor):
for attr in tensor.__tensor_flatten__()[0]:
yield from _tensors(f"{name}.{attr}", getattr(tensor, attr))
else:
yield name, tensor
def _weights(model: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
for name, param in model.named_parameters():
yield from _tensors(name, param)
def compute_tensor_digests(model: nn.Module) -> dict[str, str]:
return {
name: hashlib.sha256(
weight.detach().cpu().contiguous().view(-1).view(torch.uint8).numpy()
).hexdigest()
for name, weight in _weights(model)
}
def zero_weights(model: nn.Module) -> None:
for _, weight in _weights(model):
weight.data.zero_()
-365
View File
@@ -1,365 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2023-2025 vLLM Team
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Inference-only Arcee (AFM) model – adds support for ReLU^2 feed-forward
# activation.
from collections.abc import Iterable
from itertools import islice
from typing import Any
import torch
from torch import nn
from transformers import LlamaConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.activation import ReLUSquaredActivation
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from .interfaces import (
EagleModelMixin,
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
)
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
class ArceeMLP(nn.Module):
"""Feed-forward layer for Arcee using ReLU^2 activation
(no gating as in LLaMA)."""
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Any | None = None,
bias: bool = False,
prefix: str = "",
reduce_results: bool = True,
) -> None:
super().__init__()
# Single linear projection up to intermediate size
# (no separate gate projection)
self.up_proj = ColumnParallelLinear(
input_size=hidden_size,
output_size=intermediate_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.up_proj",
)
# Down projection back to hidden size
self.down_proj = RowParallelLinear(
input_size=intermediate_size,
output_size=hidden_size,
bias=bias,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "relu2":
raise ValueError(
f"Unsupported activation: {hidden_act}. "
"Only 'relu2' is supported for AFM."
)
# Define ReLU^2 activation: (ReLU(x))^2 elementwise
self.act_fn = ReLUSquaredActivation()
def forward(self, x: torch.Tensor) -> torch.Tensor:
x, _ = self.up_proj(x) # Project to intermediate size
x = self.act_fn(x) # Apply ReLU^2 activation elementwise
x, _ = self.down_proj(x) # Project back down to hidden size
return x
class ArceeDecoderLayer(nn.Module):
"""Transformer decoder block for Arcee, with self-attention and
ReLU^2 MLP."""
def __init__(
self,
config: LlamaConfig,
cache_config: Any | None = None,
quant_config: Any | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
# Determine if attention bias is needed (some variants use bias terms)
attention_bias = getattr(config, "attention_bias", False) or getattr(
config, "bias", False
)
bias_o_proj = attention_bias
if hasattr(config, "qkv_bias"):
attention_bias = config.qkv_bias
# Self-Attention (using LLaMA's attention structure)
from vllm.model_executor.models.llama import (
LlamaAttention, # import here to avoid circular import
)
self.self_attn = LlamaAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=getattr(
config, "num_key_value_heads", config.num_attention_heads
),
max_position_embeddings=max_position_embeddings,
quant_config=quant_config,
bias=attention_bias,
bias_o_proj=bias_o_proj,
cache_config=cache_config,
prefix=f"{prefix}.self_attn",
attn_type=getattr(
config, "attn_type", "decoder"
), # assume decoder (causal) unless specified
)
# MLP with ReLU^2 activation
self.mlp = ArceeMLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
bias=getattr(config, "mlp_bias", False),
prefix=f"{prefix}.mlp",
)
# Layer normalization layers (RMSNorm as in LLaMA)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self-Attention block
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
# Fused residual add + layernorm if supported
hidden_states, residual = self.input_layernorm(hidden_states, residual)
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
# Feed-forward block
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
@support_torch_compile
class ArceeModel(nn.Module, EagleModelMixin):
"""The transformer model backbone for Arcee (embedding layer + stacked
decoder blocks + final norm)."""
def __init__(
self,
*,
vllm_config,
prefix: str = "",
layer_type: type[nn.Module] = ArceeDecoderLayer,
) -> None:
super().__init__()
config: LlamaConfig = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.quant_config = quant_config
self.config = config
self.vocab_size = config.vocab_size
# Word embeddings (parallelized if using pipeline parallel)
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
config.hidden_size,
quant_config=quant_config,
)
else:
self.embed_tokens = PPMissingLayer() # placeholder on non-embedding ranks
# Build decoder layers across pipeline ranks
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: layer_type(
config=config,
cache_config=cache_config,
quant_config=quant_config,
prefix=prefix,
),
prefix=f"{prefix}.layers",
)
# Final RMSNorm on the last pipeline stage
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
# Prepare factory for empty intermediate tensors
# (for pipeline scheduling)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
# Embedding lookup (on first pipeline rank)
if get_pp_group().is_first_rank:
hidden_states = (
inputs_embeds
if inputs_embeds is not None
else self.embed_input_ids(input_ids)
)
residual = None
else:
assert intermediate_tensors is not None, (
"IntermediateTensors must be provided for non-first pipeline ranks"
)
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
):
hidden_states, residual = layer(positions, hidden_states, residual)
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, residual
)
if not get_pp_group().is_last_rank:
# Send intermediate results to the next pipeline stage
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
# On last rank: apply final layer norm
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
class ArceeForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
):
"""Arcee Model for causal language modeling, integrated with vLLM
runtime."""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
},
orig_to_new_substr={
"gate_proj": None,
},
)
# Map fused module names to their submodule components
# (for quantization and LoRA)
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
def __init__(self, *, vllm_config, prefix: str = "") -> None:
super().__init__()
config = vllm_config.model_config.hf_config
self.config = config
# Initialize the inner Transformer model (ArceeModel)
self.model = ArceeModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
)
# On the last pipeline stage, set up the LM head and logits processor
if get_pp_group().is_last_rank:
# Determine vocabulary size (including any LoRA extra tokens
# for padded LM head)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=vllm_config.quant_config,
bias=getattr(config, "lm_head_bias", False),
prefix=maybe_prefix(prefix, "lm_head"),
)
if config.tie_word_embeddings:
# Tie output weights with input embedding matrix
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
logit_scale = getattr(config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(
config.vocab_size, scale=logit_scale
)
else:
# Placeholder for lm_head on non-last ranks
self.lm_head = PPMissingLayer()
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
return model_output
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
# Compute final logits from hidden states (last pipeline rank only)
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights into the model (delegates to inner model and handles
tied embeddings)."""
loader = AutoWeightsLoader(self)
# AutoWeightLoader handles weight name remapping, including fusing
# separate q_proj, k_proj, v_proj into qkv_proj
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+9 -4
View File
@@ -461,6 +461,7 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
"""
is_pooling_model = True
embedding_class: type[nn.Module] = BertEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -504,7 +505,7 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
def _build_model(self, vllm_config: VllmConfig, prefix: str = "") -> BertModel:
return BertModel(
vllm_config=vllm_config, prefix=prefix, embedding_class=BertEmbedding
vllm_config=vllm_config, prefix=prefix, embedding_class=self.embedding_class
)
def _build_pooler(self, pooler_config: PoolerConfig) -> Pooler:
@@ -781,6 +782,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
"""
is_pooling_model = True
embedding_class: type[nn.Module] = BertEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -790,7 +792,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
self.bert = BertPoolingModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "bert"),
embedding_class=BertEmbedding,
embedding_class=self.embedding_class,
)
self.classifier = nn.Linear(
config.hidden_size,
@@ -840,6 +842,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
@default_pooling_type(tok_pooling_type="ALL")
class BertForTokenClassification(nn.Module):
is_pooling_model = True
embedding_class: type[nn.Module] = BertEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -849,7 +852,7 @@ class BertForTokenClassification(nn.Module):
self.bert = BertModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "bert"),
embedding_class=BertEmbedding,
embedding_class=self.embedding_class,
)
self.classifier = nn.Linear(
config.hidden_size, config.num_labels, dtype=self.head_dtype
@@ -927,13 +930,15 @@ class BertForMaskedLM(nn.Module):
}
)
embedding_class: type[nn.Module] = BertEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.bert = BertModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "bert"),
embedding_class=BertEmbedding,
embedding_class=self.embedding_class,
)
self.mlm_head = BertMLMHead(
hidden_size=config.hidden_size,
@@ -572,7 +572,6 @@ class GteNewModel(BertWithRope):
super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
# GteNewModel only gate_up_proj does not have bias.
# Hack method learned from vllm/model_executor/models/glm.py
for layer in self.encoder.layers:
layer.mlp.gate_up_proj.bias = None
layer.mlp.gate_up_proj.skip_bias_add = True
+3 -14
View File
@@ -646,20 +646,9 @@ class MambaModelConfig(VerifyAndUpdateConfig):
cache_config.mamba_cache_mode,
model_config.architecture,
)
if (
cache_config.mamba_cache_mode == "all"
and not model_config.supports_mamba_prefix_caching
):
cache_config.mamba_cache_mode = "align"
logger.warning(
"Hybrid or mamba-based model detected without support "
"for prefix caching with Mamba cache 'all' mode: "
"falling back to 'align' mode."
)
if cache_config.mamba_cache_mode == "align":
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
# By default, mamba block size will be set to max_model_len (see
# below). When enabling prefix caching, we align mamba block size
# to the block size as the basic granularity for prefix caching.
+2 -1
View File
@@ -658,6 +658,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_dim,
dtype=self.dtype,
cache_role=SparseCacheRole.INDEXER,
@@ -715,7 +716,7 @@ class Indexer(nn.Module):
disable_tp=True,
prefix=f"{prefix}.wk_weights_proj",
)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32)
self.softmax_scale = self.head_dim**-0.5
self.scale_fmt = "ue8m0"
@@ -909,9 +909,12 @@ class DiffusionGemmaModelState(ModelState):
self._req_id_to_index: dict[str, int] = {}
# Persistent buffer for per-request causal flags, updated in-place
# so FULL CUDA graph replay sees the latest values.
# so FULL CUDA graph replay sees the latest values. Must be int32:
# FlashAttentionMetadataBuilder.build() rejects other dtypes, since
# an out-of-place cast there would detach the captured graph from
# this buffer and freeze replay at the capture-time snapshot.
self._causal_buf = torch.zeros(
self.max_num_reqs, dtype=torch.bool, device=device
self.max_num_reqs, dtype=torch.int32, device=device
)
# Persistent inputs_embeds buffer — required so FULL CUDA graph
@@ -1116,11 +1119,11 @@ class DiffusionGemmaModelState(ModelState):
# Invariant: the sampler flips is_encoder_phase to False only after a
# request's FINAL prompt chunk, so a prompt spanning multiple chunks
# (longer than the token budget) stays causal for every chunk.
self._causal_buf[:actual_num_reqs] = self.diffusion_states.is_encoder_phase[
slots
]
self._causal_buf[:actual_num_reqs].copy_(
self.diffusion_states.is_encoder_phase[slots]
)
if actual_num_reqs < num_reqs:
self._causal_buf[actual_num_reqs:num_reqs] = False
self._causal_buf[actual_num_reqs:num_reqs] = 0
causal: bool | torch.Tensor = self._causal_buf[:num_reqs]
return build_attn_metadata(
-24
View File
@@ -1,24 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only HF format GLM-4 model compatible with THUDM weights."""
from vllm.config import VllmConfig
from vllm.model_executor.models.llama import LlamaForCausalLM
from .utils import PPMissingLayer
class GlmForCausalLM(LlamaForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
hf_config = vllm_config.model_config.hf_config
hf_config.rope_parameters["partial_rotary_factor"] = 0.5
super().__init__(vllm_config=vllm_config, prefix=prefix)
# Hack Llama model to fit HF format GLM implementation
# Attention difference between GLM and Llama:
# 1. Half partial rotary_dim and no Neox style.
# 2. There is no bias for o_proj in attention
for layer in self.model.layers:
if not isinstance(layer, PPMissingLayer):
layer.self_attn.rotary_emb.is_neox_style = False
layer.self_attn.o_proj.bias = None
layer.self_attn.o_proj.skip_bias_add = True
+16 -7
View File
@@ -61,6 +61,10 @@ from vllm.model_executor.layers.attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import (
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -616,6 +620,7 @@ class Glm4vVisionTransformer(nn.Module):
vision_config: Glm4vVisionConfig,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -642,6 +647,7 @@ class Glm4vVisionTransformer(nn.Module):
in_channels=in_channels,
hidden_size=self.hidden_size,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
norm_layer = partial(RMSNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
@@ -924,7 +930,7 @@ class Glm4vVisionTransformer(nn.Module):
encoder_metadata = self.prepare_encoder_metadata(grid_thw)
# patchify
x = x.to(device=self.device, dtype=self.dtype)
x = self.input_norm(x.to(device=self.device), self.dtype)
x = self.patch_embed(x)
x = self.post_conv_layernorm(x)
@@ -1755,6 +1761,7 @@ class Glm4vForConditionalGeneration(
supports_encoder_tp_data = True
supports_tower_connector_lora = True
supports_mm_device_do_normalize = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
@@ -1786,6 +1793,7 @@ class Glm4vForConditionalGeneration(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
@@ -1867,7 +1875,7 @@ class Glm4vForConditionalGeneration(
if image_input["type"] == "image_embeds":
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
else:
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
pixel_values = image_input["pixel_values"]
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
@@ -1888,9 +1896,7 @@ class Glm4vForConditionalGeneration(
if video_input["type"] == "video_embeds":
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
else:
pixel_values_videos = video_input["pixel_values_videos"].type(
self.visual.dtype
)
pixel_values_videos = video_input["pixel_values_videos"]
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual,
@@ -2098,8 +2104,11 @@ class Glm4vForConditionalGeneration(
flattened_patch_size = (
in_channels * temporal_patch_size * patch_size * patch_size
)
dummy_pixel_values = torch.randn(
total_patches, flattened_patch_size, device=device, dtype=dtype
dummy_pixel_values = torch.zeros(
total_patches,
flattened_patch_size,
device=device,
dtype=self.visual.input_norm.input_dtype or dtype,
)
# Override max_seqlen with a safe upper bound for capture.
+12 -2
View File
@@ -48,6 +48,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -254,9 +255,17 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
vision_config: "GlmOcrVisionConfig",
norm_eps: float = 1e-5,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__(text_config, vision_config, norm_eps, quant_config, prefix)
super().__init__(
text_config,
vision_config,
norm_eps,
quant_config,
input_norm,
prefix,
)
del self.post_conv_layernorm
del self.embeddings
@@ -334,7 +343,7 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
# patchify
x = x.to(device=self.device, dtype=self.dtype)
x = self.input_norm(x.to(device=self.device), self.dtype)
x = self.patch_embed(x)
# compute position embedding
@@ -393,5 +402,6 @@ class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration):
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
+4 -3
View File
@@ -35,6 +35,7 @@ from vllm.distributed.parallel_state import (
)
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -156,11 +157,11 @@ class GPT2Block(nn.Module):
hidden_size = config.hidden_size
inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_1 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.attn = GPT2Attention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_2 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.mlp = GPT2MLP(inner_dim, config, quant_config, prefix=f"{prefix}.mlp")
def forward(
@@ -212,7 +213,7 @@ class GPT2Model(nn.Module):
lambda prefix: GPT2Block(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
+3 -2
View File
@@ -31,6 +31,7 @@ from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -163,7 +164,7 @@ class GPTJBlock(nn.Module):
):
super().__init__()
inner_dim = 4 * config.n_embd if config.n_inner is None else config.n_inner
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.ln_1 = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.attn = GPTJAttention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
@@ -206,7 +207,7 @@ class GPTJModel(nn.Module):
lambda prefix: GPTJBlock(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
+4 -1
View File
@@ -76,6 +76,8 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
"w2_bias": "gpt_oss_w2",
"w13_weight_scale": "gpt_oss_w13",
"w2_weight_scale": "gpt_oss_w2",
"w13_weight_packed": "gpt_oss_w13",
"w2_weight_packed": "gpt_oss_w2",
}
@@ -263,7 +265,7 @@ class GptOssRoutedExperts(RoutedExperts):
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
is_w13 = shard_id == "gpt_oss_w13"
is_weight = weight_name.endswith("_weight")
is_weight = weight_name.endswith(("_weight", "_weight_packed"))
is_partitioned_scale = weight_name.endswith("_weight_scale")
if is_weight:
@@ -347,6 +349,7 @@ class GptOssRoutedExperts(RoutedExperts):
) or quant_method_name in (
"CompressedTensorsW4A4Nvfp4MoEMethod",
"Nvfp4OnlineMoEMethod",
"CompressedTensorsWNA16MoEMethod",
):
self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id)
else:
+3 -2
View File
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
from vllm.tokenizers import TokenizerLike
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
@@ -1450,7 +1451,7 @@ class SupportsTranscription(Protocol):
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: object,
tokenizer: "TokenizerLike",
) -> str:
"""Parse the detected language from model output token IDs.
@@ -1462,7 +1463,7 @@ class SupportsTranscription(Protocol):
@classmethod
def get_language_token_ids(
cls,
tokenizer: object,
tokenizer: "TokenizerLike",
) -> list[int] | None:
"""Return token IDs that represent valid language tokens.
@@ -418,12 +418,6 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLMBase):
self.quant_config = vllm_config.quant_config
self.scheduler_config = vllm_config.scheduler_config
if vllm_config.cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Intern-S2-Mobius does not support 'all' prefix caching; "
"use '--mamba-cache-mode=align' instead."
)
nn.Module.__init__(self)
self.config = config
self.model = InternS2MobiusModel(
+237 -95
View File
@@ -71,6 +71,7 @@ from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -119,6 +120,96 @@ def interleaved_to_split(x):
return x.reshape(*x.shape[:-1], -1, 2).transpose(-1, -2).reshape(*x.shape[:-1], -1)
def is_rope_weights_folding_supported(
qk_proj: nn.Module,
dual_chunk_attention_config: dict[str, Any] | None,
) -> bool:
"""Whether the partial-RoPE weight-fold path is safe for this attention.
Folding permutes the q/k projection's output channels at load time, which
only produces correct numerics when:
* the q/k projection is dense/unquantized and
* dual-chunk attention is disabled -- the P.RoPE equivalence has not been
validated with dual-chunk attention.
When either condition fails we fall back to the runtime channel-permutation
RoPE path instead of folding.
"""
is_dense_qk_proj = isinstance(
getattr(qk_proj, "quant_method", None), UnquantizedLinearMethod
)
return is_dense_qk_proj and dual_chunk_attention_config is None
def apply_partial_rope(
rotary_emb,
positions: torch.Tensor,
q: torch.Tensor,
k: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_head_dim: int,
fold_rope_weights: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
if fold_rope_weights or rope_head_dim == head_dim:
return rotary_emb(positions, q, k)
q = q.reshape(*q.shape[:-1], num_heads, head_dim)
k = k.reshape(*k.shape[:-1], num_kv_heads, head_dim)
q_rope, q_nope = torch.split(
split_to_interleaved(q),
split_size_or_sections=[rope_head_dim, head_dim - rope_head_dim],
dim=-1,
)
k_rope, k_nope = torch.split(
split_to_interleaved(k),
split_size_or_sections=[rope_head_dim, head_dim - rope_head_dim],
dim=-1,
)
q_rope, k_rope = rotary_emb(
positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope)
)
q = interleaved_to_split(
torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1)
).reshape(*q.shape[:-2], -1)
k = interleaved_to_split(
torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1)
).reshape(*k.shape[:-2], -1)
return q, k
def _rope_weight_perm(head_dim: int, rope_head_dim: int) -> torch.Tensor:
"""Per-head gather index that folds K2Horizon's partial-RoPE channel reordering.
The K2Horizon checkpoint stores each head's channels in a GPT-J-interleaved
convention with rope/nope channels interleaved. Applying this fixed
(position-independent) permutation to the q/k weight rows lets the runtime
use vLLM's *native* NeoX partial-RoPE path (``head_size = head_dim``,
``rotary_dim = rope_head_dim``) with no per-forward permutes/all-gather.
"""
D, R = head_dim, rope_head_dim
assert R % 2 == 0 and R <= D, (
f"partial NeoX RoPE requires an even rope_head_dim <= head_dim, "
f"got R={R}, D={D}"
)
h = D // 2
idx = torch.cat(
[
torch.arange(0, R // 2), # rope first-half channels
torch.arange(h, h + R // 2), # rope second-half channels
torch.arange(R // 2, h), # nope remainder (first half)
torch.arange(h + R // 2, D), # nope remainder (second half)
]
)
return idx
def calc_router_weights(
router_logits: torch.Tensor,
e_score_correction_bias: torch.Tensor | None,
@@ -411,6 +502,7 @@ class K2HorizonSparseMoeBlock(nn.Module):
self.gate.bias = nn.Parameter(self.gate.bias.float(), requires_grad=False)
self.num_shared_experts = config.num_shared_experts
self.shared_experts: K2HorizonMLP | None
if config.num_shared_experts > 0:
self.shared_experts = K2HorizonMLP(
hidden_size=config.hidden_size,
@@ -529,11 +621,37 @@ class K2HorizonAttention(nn.Module):
)
self.rope_head_dim = rope_head_dim or self.head_dim
self.rotary_emb = get_rope(
self.rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
# Fold the partial-RoPE channel permutation into the q/k weights only
# when supported (dense projections + no dual-chunk attention);
# otherwise keep the runtime channel-permutation path
# (see ``is_rope_weights_folding_supported``).
self.fold_rope_weights = is_rope_weights_folding_supported(
self.qkv_proj, dual_chunk_attention_config
)
if self.fold_rope_weights:
rope_parameters = dict(rope_parameters)
rope_parameters["rope_dim"] = self.rope_head_dim
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
is_neox_style=True,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
else:
self.rotary_emb = get_rope(
self.rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
attn_kwargs: dict[str, Any] = (
{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {}
)
self.attn = Attention(
self.num_heads,
@@ -543,12 +661,7 @@ class K2HorizonAttention(nn.Module):
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
**attn_kwargs,
)
self.query_key_norm = query_key_norm
@@ -584,40 +697,17 @@ class K2HorizonAttention(nn.Module):
q = self.q_norm(q)
k = self.k_norm(k)
if self.rope_head_dim == self.head_dim:
q, k = self.rotary_emb(positions, q, k)
else:
q = q.reshape(*q.shape[:-1], self.num_heads, self.head_dim)
k = k.reshape(*k.shape[:-1], self.num_kv_heads, self.head_dim)
q_rope, q_nope = torch.split(
split_to_interleaved(q),
split_size_or_sections=[
self.rope_head_dim,
self.head_dim - self.rope_head_dim,
],
dim=-1,
)
k_rope, k_nope = torch.split(
split_to_interleaved(k),
split_size_or_sections=[
self.rope_head_dim,
self.head_dim - self.rope_head_dim,
],
dim=-1,
)
q_rope, k_rope = self.rotary_emb(
positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope)
)
q = interleaved_to_split(
torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1)
).reshape(*q.shape[:-2], -1)
k = interleaved_to_split(
torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1)
).reshape(*k.shape[:-2], -1)
q, k = apply_partial_rope(
self.rotary_emb,
positions,
q,
k,
self.num_heads,
self.num_kv_heads,
self.head_dim,
self.rope_head_dim,
self.fold_rope_weights,
)
attn_output = self.attn(q, k, v)
@@ -732,11 +822,37 @@ class K2HorizonMoVAAttention(nn.Module):
)
self.rope_head_dim = rope_head_dim or self.head_dim
self.rotary_emb = get_rope(
self.rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
# Fold the partial-RoPE channel permutation into the q/k weights only
# when supported (dense projections + no dual-chunk attention);
# otherwise keep the runtime channel-permutation path
# (see ``is_rope_weights_folding_supported``).
self.fold_rope_weights = is_rope_weights_folding_supported(
self.qk_proj, dual_chunk_attention_config
)
if self.fold_rope_weights:
rope_parameters = dict(rope_parameters)
rope_parameters["rope_dim"] = self.rope_head_dim
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
is_neox_style=True,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
else:
self.rotary_emb = get_rope(
self.rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
attn_kwargs: dict[str, Any] = (
{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {}
)
self.attn = Attention(
self.num_heads,
@@ -746,12 +862,7 @@ class K2HorizonMoVAAttention(nn.Module):
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
**attn_kwargs,
)
self.query_key_norm = query_key_norm
@@ -838,40 +949,17 @@ class K2HorizonMoVAAttention(nn.Module):
q = self.q_norm(q)
k = self.k_norm(k)
if self.rope_head_dim == self.head_dim:
q, k = self.rotary_emb(positions, q, k)
else:
q = q.reshape(*q.shape[:-1], self.num_heads, self.head_dim)
k = k.reshape(*k.shape[:-1], self.num_kv_heads, self.head_dim)
q_rope, q_nope = torch.split(
split_to_interleaved(q),
split_size_or_sections=[
self.rope_head_dim,
self.head_dim - self.rope_head_dim,
],
dim=-1,
)
k_rope, k_nope = torch.split(
split_to_interleaved(k),
split_size_or_sections=[
self.rope_head_dim,
self.head_dim - self.rope_head_dim,
],
dim=-1,
)
q_rope, k_rope = self.rotary_emb(
positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope)
)
q = interleaved_to_split(
torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1)
).reshape(*q.shape[:-2], -1)
k = interleaved_to_split(
torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1)
).reshape(*k.shape[:-2], -1)
q, k = apply_partial_rope(
self.rotary_emb,
positions,
q,
k,
self.num_heads,
self.num_kv_heads,
self.head_dim,
self.rope_head_dim,
self.fold_rope_weights,
)
attn_output = self.attn(q, k, v)
@@ -1109,6 +1197,27 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
("gate_up_proj", "up_proj", 1),
]
# Per-head gather index used to fold K2Horizon's partial-RoPE channel
# reordering into the q_proj/k_proj rows and the q_norm/k_norm scales,
# so the runtime can use vLLM's native partial NeoX RoPE.
head_dim = self.config.head_dim or (
self.config.hidden_size // self.config.num_attention_heads
)
rope_head_dim = self.config.rope_head_dim or head_dim
rope_perm_idx = _rope_weight_perm(head_dim, rope_head_dim)
# Attention modules whose q/k projections are dense: only these fold the
# RoPE channel permutation into their q/k weights, biases and q/k norm
# scales at load time. Attentions with packed/quantized q/k projections
# keep the runtime channel-permutation path, so their weights must be
# loaded verbatim (folding would misalign per-channel scales).
rope_fold_prefixes = tuple(
f"{module_name}."
for module_name, module in self.named_modules()
if isinstance(module, (K2HorizonAttention, K2HorizonMoVAAttention))
and getattr(module, "fold_rope_weights", False)
)
# Skip loading extra parameters for GPTQ/modelopt models.
# ignore_suffixes = (
# ".bias",
@@ -1126,9 +1235,10 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
if "scale" in name or "zero_point" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
# QK norm weights
if name.endswith(".self_attn.q_norm.weight") or name.endswith(
@@ -1139,6 +1249,14 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
if name not in params_dict:
continue
# Fold the per-head RoPE channel permutation into the norm scales.
if self.config.query_key_norm and name.startswith(rope_fold_prefixes):
loaded_weight = (
loaded_weight.view(-1, head_dim)[:, rope_perm_idx]
.reshape(-1)
.contiguous()
)
param = params_dict[name]
tp_rank = get_tensor_model_parallel_rank()
tp_size = get_tensor_model_parallel_world_size()
@@ -1157,7 +1275,9 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
kv_rank
]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader: Callable[..., Any] = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
continue
@@ -1203,6 +1323,27 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
if ".self_attn." in name and param_name == "gate_up_proj":
assert weight_name == "gate_proj"
continue
# Fold the per-head RoPE channel permutation into the dense
# q_proj / k_proj rows (W' = P W), and biases, before TP
# sharding.
if shard_id in ("q", "k") and name.startswith(rope_fold_prefixes):
if name.endswith(".weight"):
hidden = loaded_weight.shape[-1]
loaded_weight = (
loaded_weight.view(-1, head_dim, hidden)[
:, rope_perm_idx, :
]
.reshape(-1, hidden)
.contiguous()
)
elif self.config.attention_bias and name.endswith(".bias"):
loaded_weight = (
loaded_weight.view(-1, head_dim)[:, rope_perm_idx]
.reshape(-1)
.contiguous()
)
if (
".self_attn." in name
and param_name == "qkv_proj"
@@ -1249,9 +1390,10 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
continue
if name.endswith("scale"):
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
if name not in params_dict:
continue
+16 -15
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from functools import partial
from typing import Annotated, Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias, cast
import numpy as np
import regex as re
@@ -288,7 +288,7 @@ class CustomQwen2VLVE(Qwen2VisionTransformer):
"""Drop-in replacement for the HF `_from_config` constructor."""
return cls(config)
def forward(
def forward( # type: ignore[override]
self,
pixel_values: torch.Tensor,
grid_thw: torch.Tensor,
@@ -331,12 +331,12 @@ class CustomQwen2VLVE(Qwen2VisionTransformer):
# Pre-compute seqlens for attention backend.
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
encoder_states = () if output_hidden_states else None
encoder_states: tuple[torch.Tensor, ...] = ()
for blk in self.blocks:
if output_hidden_states:
# Store patch-level states (S, D).
encoder_states = encoder_states + (x.squeeze(1),)
encoder_states += (x.squeeze(1),)
x = blk(
x,
@@ -348,14 +348,15 @@ class CustomQwen2VLVE(Qwen2VisionTransformer):
# Final hidden state at patch level (S, D).
hidden_states = x.squeeze(1)
all_hidden_states = None
if output_hidden_states:
encoder_states = encoder_states + (hidden_states,)
all_hidden_states = encoder_states + (hidden_states,)
if not return_dict:
return tuple(v for v in [hidden_states, encoder_states] if v is not None)
return tuple(v for v in [hidden_states, all_hidden_states] if v is not None)
return BaseModelOutput(
last_hidden_state=hidden_states,
hidden_states=encoder_states,
hidden_states=all_hidden_states,
)
def get_num_tokens(self) -> int:
@@ -479,7 +480,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
assert isinstance(prompt_text, str)
# Images
image_inputs = hf_data.get("images", [])
image_inputs = cast(list[Any], hf_data.get("images", []))
pixel_sizes = []
if not isinstance(image_inputs[0], Image.Image):
image_inputs = [Image.fromarray(image) for image in image_inputs]
@@ -487,9 +488,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
image_processor = self.info.get_hf_processor().image_processor
processor_output = [image_processor(image) for image in image_inputs]
pixel_values = [o["pixel_values"] for o in processor_output]
image_meta = [o["image_meta"] for o in processor_output]
image_meta_list = [o["image_meta"] for o in processor_output]
# list of dict -> dict of list
image_meta = {k: [d[k] for d in image_meta] for k in image_meta[0]}
image_meta = {k: [d[k] for d in image_meta_list] for k in image_meta_list[0]}
for pixel_value in pixel_values:
pixel_sizes.append(pixel_value.shape[0])
@@ -545,7 +546,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
def get_replacement(idx: int) -> Sequence[int]:
def get_replacement(idx: int) -> list[int]:
out_item = out_mm_kwargs["image"][idx]
image_token_thw = out_item["image_token_thw"].data
assert isinstance(image_token_thw, torch.Tensor)
@@ -683,7 +684,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
def _get_visual_feature_at(
self,
v_output: Sequence[torch.Tensor],
layer_index: int | Sequence[int],
layer_index: int | list[int] | tuple[int, ...],
) -> torch.Tensor:
if isinstance(layer_index, (list, tuple)):
visual_features = torch.stack(v_output, dim=1)[
@@ -696,7 +697,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
def forward_vision(
self,
pixel_values: torch.Tensor,
image_metas: dict | None = None,
image_metas: dict,
) -> torch.Tensor:
vision_model_args = {
"pixel_values": pixel_values,
@@ -714,7 +715,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
def forward_projector(
self,
visual_features: torch.Tensor,
image_metas: dict | None = None,
image_metas: dict,
) -> torch.Tensor:
visual_embeds = self.abstractor(
visual_features,
@@ -725,7 +726,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
def forward_and_project_vision(
self,
pixel_values: torch.Tensor,
image_metas: dict | None = None,
image_metas: dict,
) -> torch.Tensor:
assert pixel_values is not None
visual_features = self.forward_vision(pixel_values, image_metas=image_metas)
+29 -34
View File
@@ -208,8 +208,8 @@ class KeyeVisionEmbeddings(nn.Module):
self.num_patches = (self.image_size // self.patch_size) ** 2
self.num_positions = self.num_patches
self.cache_position_embedding = dict()
self.cache_position_count = dict()
self.cache_position_embedding: dict[tuple[int, int], torch.Tensor] = {}
self.cache_position_count: dict[tuple[int, int], int] = {}
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
self.packing_position_embedding = nn.Embedding(32768, self.embed_dim)
@@ -264,7 +264,7 @@ class KeyeVisionEmbeddings(nn.Module):
if len(self.cache_position_embedding) >= max_cache:
min_hit_grid = min(
self.cache_position_count,
key=self.cache_position_count.get,
key=self.cache_position_count.__getitem__,
)
self.cache_position_count.pop(min_hit_grid)
self.cache_position_embedding.pop(min_hit_grid)
@@ -278,8 +278,7 @@ class KeyeVisionEmbeddings(nn.Module):
self,
pixel_values: torch.FloatTensor,
position_ids: torch.Tensor | None = None,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
image_grid_thw: list[tuple[int, int, int]] | None = None,
interpolate_pos_encoding=False,
) -> torch.Tensor:
if pixel_values.dim() == 4:
@@ -408,7 +407,7 @@ class KeyeSiglipAttention(nn.Module):
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
output_attentions: bool | None = False,
cu_seqlens: list[torch.Tensor] | None = None,
cu_seqlens: torch.Tensor | None = None,
rope_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
@@ -417,6 +416,8 @@ class KeyeSiglipAttention(nn.Module):
dim=-1,
)
if cu_seqlens is None:
raise ValueError("cu_seqlens cannot be None.")
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
if rope_emb is None:
@@ -432,8 +433,6 @@ class KeyeSiglipAttention(nn.Module):
self.head_dim,
)
else:
if cu_seqlens is None:
raise ValueError("cu_seqlens cannot be None when rope_emb is not None.")
cos, sin = rope_emb
q = q.view(*q.shape[:-1], self.num_heads, self.head_dim)
k = k.view(
@@ -511,7 +510,7 @@ class KeyeSiglipEncoderLayer(nn.Module):
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
output_attentions: bool | None = False,
cu_seqlens: list[torch.Tensor] | None = None,
cu_seqlens: torch.Tensor | None = None,
rope_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> tuple[torch.FloatTensor]:
residual = hidden_states
@@ -576,13 +575,12 @@ class KeyeSiglipEncoder(nn.Module):
attention_mask: torch.Tensor | None = None,
output_attentions: bool | None = None,
output_hidden_states: bool | None = None,
cu_seqlens: list[torch.Tensor] | None = None,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
cu_seqlens: torch.Tensor | None = None,
image_grid_thw: list[tuple[int, int, int]] | None = None,
height_position_ids: torch.Tensor | None = None,
width_position_ids: torch.Tensor | None = None,
use_rope: bool | None = False,
window_size: bool | None = -1,
window_size: int = -1,
vision_or_text: str = "vision",
) -> BaseModelOutput:
device = inputs_embeds.device
@@ -662,14 +660,13 @@ class KeyeSiglipVisionTransformer(nn.Module):
position_ids: torch.Tensor | None = None,
height_position_ids: torch.Tensor | None = None,
width_position_ids: torch.Tensor | None = None,
cu_seqlens: list[torch.Tensor] | None = None,
cu_seqlens: torch.Tensor | None = None,
padding_mask: torch.Tensor | None = None,
vision_return_embed_list: bool | None = False,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
image_grid_thw: list[tuple[int, int, int]] | None = None,
return_pooler_output: bool | None = True,
use_rope: bool | None = False,
window_size: bool | None = -1,
window_size: int = -1,
) -> BaseModelOutputWithPooling:
hidden_states = self.embeddings(
pixel_values,
@@ -757,12 +754,11 @@ class KeyeSiglipVisionModel(nn.Module):
interpolate_pos_encoding: bool = False,
position_ids: torch.Tensor | None = None,
vision_return_embed_list: bool | None = False,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
cu_seqlens: list[torch.Tensor] | None = None,
image_grid_thw: list[tuple[int, int, int]] | None = None,
cu_seqlens: torch.Tensor | None = None,
return_pooler_output: bool | None = True,
use_rope: bool | None = False,
window_size: bool | None = -1,
window_size: int = -1,
) -> BaseModelOutputWithPooling:
return self.vision_model(
pixel_values=pixel_values,
@@ -1337,7 +1333,7 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
def _process_video_embeds(
self,
video_type: Literal["video_embeds", "pixel_values_videos"],
video_grid_thw: list[torch.Tensor],
video_grid_thw: torch.Tensor,
pixel_values_videos: torch.Tensor | None = None,
) -> torch.Tensor | list[torch.Tensor]:
siglip_position_ids = list()
@@ -1361,6 +1357,7 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
"Video embeddings are not supported for this processing path."
)
else:
assert pixel_values_videos is not None
pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
# These are host-built; concat straight into pinned buffers so
# the H2D copies stay non-blocking.
@@ -1535,12 +1532,11 @@ class KeyeForConditionalGeneration(
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
return KeyeImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
return KeyeImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
def _parse_and_validate_video_input(
self, **kwargs: object
@@ -1559,12 +1555,11 @@ class KeyeForConditionalGeneration(
video_grid_thw=video_grid_thw,
)
if video_embeds is not None:
return KeyeVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
)
return KeyeVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
)
def _process_video_input(
self, video_input: KeyeVideoInputs
+13 -15
View File
@@ -75,7 +75,7 @@ def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
def get_num_patches(
grid_thw: torch.Tensor, num_frames: list[int] | torch.Tensor
) -> list[int]:
) -> torch.Tensor:
"""Return num_patches per video.
Args:
@@ -84,7 +84,7 @@ def get_num_patches(
num_frames: List or tensor indicating the number of frames per video
Returns:
List of ints representing the number of patches for each video
Tensor of the number of patches for each video
Examples:
>>> # Suppose there are 2 videos with a total of 3 grids
@@ -538,12 +538,11 @@ class KeyeVL1_5ForConditionalGeneration(
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
return KeyeVL1_5ImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
return KeyeVL1_5ImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
def _parse_and_validate_video_input(
self, **kwargs: object
@@ -564,13 +563,12 @@ class KeyeVL1_5ForConditionalGeneration(
num_frames=num_frames,
)
if video_embeds is not None:
return KeyeVL1_5VideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
num_frames=num_frames,
)
return KeyeVL1_5VideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
num_frames=num_frames,
)
def _process_video_input(
self, video_input: KeyeVL1_5VideoInputs
+3 -1
View File
@@ -18,6 +18,7 @@ from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import PromptType, TokensPrompt
from vllm.model_executor.model_loader import DefaultModelLoader
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
SupportsTranscription,
@@ -240,6 +241,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
# KimiAudioProcessor expects raw numpy arrays
mm_data = hf_inputs.hf_data
if audios := mm_data.pop("audio", []):
assert isinstance(audios, Sequence)
audio_arrays = []
for aud in audios:
if isinstance(aud, (tuple, list)) and len(aud) == 2:
@@ -485,7 +487,7 @@ class KimiAudioForConditionalGeneration(
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: tuple[torch.Tensor, ...] | None = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
+12 -6
View File
@@ -136,9 +136,9 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
# may remap token IDs vs config.json.
config_token_id = hf_config.media_placeholder_token_id
resolved_token_id = tokenizer.convert_tokens_to_ids("<|media_pad|>")
unk_token_id = getattr(tokenizer, "unk_token_id", None)
is_valid_resolved = isinstance(resolved_token_id, int) and (
tokenizer.unk_token_id is None
or resolved_token_id != tokenizer.unk_token_id
unk_token_id is None or resolved_token_id != unk_token_id
)
if is_valid_resolved and resolved_token_id != config_token_id:
logger.warning_once(
@@ -165,7 +165,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
)
self.media_tokens_calculator = image_processor.media_tokens_calculator
def get_hf_processor(self):
def get_hf_processor(self, **kwargs: object):
return self.hf_processor
def get_hf_config(self):
@@ -191,7 +191,11 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
)
video_chunk_dummy_item = VisionChunkVideo(
type="video_chunk", video_chunk=dummy_videos
type="video_chunk",
video_chunk=dummy_videos,
uuid=None,
prompt="",
video_idx=0,
)
video_chunk_num_tokens = self.info.media_tokens_calculator(
video_chunk_dummy_item
@@ -204,6 +208,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
width=MaxImageTokenMeta.width,
num_images=1,
)[0],
uuid=None,
)
image_num_tokens = self.info.media_tokens_calculator(image_dummy_item)
# return the larger one
@@ -377,7 +382,7 @@ class KimiK25ForConditionalGeneration(
)
self.media_placeholder: int = self.config.media_placeholder_token_id
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig | None):
if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig):
return None
return quant_config
@@ -392,6 +397,7 @@ class KimiK25ForConditionalGeneration(
if isinstance(pixel_values, list):
pixel_values = torch.cat(pixel_values, dim=0)
assert isinstance(pixel_values, torch.Tensor)
if len(pixel_values.shape) == 5 or len(pixel_values.shape) == 3:
pixel_values = pixel_values.reshape(
@@ -612,7 +618,7 @@ class KimiK25ForConditionalGeneration(
grid_thw_list = [[1, ho * kh, wo * kw] for _ in range(max_batch_size)]
ps = self.vision_tower.patch_size
ps: int | tuple[int, int] = self.vision_tower.patch_size
if isinstance(ps, int):
ps = (ps, ps)
total_patches = max_batch_size * ho * kh * wo * kw
+1 -1
View File
@@ -804,7 +804,7 @@ class MoonViT3dPretrainedModel(nn.Module):
self,
grid_thw_list: list[list[int]],
*,
max_batch_size: int,
max_batch_size: int | None,
max_seqlen_override: int | None = None,
device: torch.device,
) -> dict[str, torch.Tensor | None]:
+3 -2
View File
@@ -268,6 +268,7 @@ class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
assert isinstance(images, ImageProcessorItems)
image_size = images.get_image_size(item_idx)
num_image_tokens = self.info.get_num_image_tokens(
image_width=image_size.width,
@@ -317,7 +318,7 @@ class KimiVLForConditionalGeneration(
assert isinstance(config.vision_config, MoonViTConfig)
self.use_data_parallel = (
model_config.multimodal_config.mm_encoder_tp_mode == "data"
model_config.get_multimodal_config().mm_encoder_tp_mode == "data"
)
self.hidden_size = config.text_config.hidden_size
@@ -469,7 +470,7 @@ class KimiVLForConditionalGeneration(
)
grid_hws_list = [(ho * kh, wo * kw) for _ in range(max_batch_size)]
patch_size = self.config.vision_config.patch_size
patch_size: int | tuple[int, int] = self.config.vision_config.patch_size
if isinstance(patch_size, int):
patch_size = (patch_size, patch_size)
-7
View File
@@ -458,13 +458,6 @@ class Lfm2ForCausalLM(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Lfm2 currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
super().__init__()
self.config = config
self.model = Lfm2Model(
-7
View File
@@ -573,13 +573,6 @@ class Lfm2MoeForCausalLM(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Lfm2Moe currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
super().__init__()
self.config = config
+2
View File
@@ -115,6 +115,8 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
dummy_inputs=Mistral3DummyInputsBuilder,
)
class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
supports_mm_device_do_normalize = False
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"model.vision_encoder.": "vision_tower.",
-253
View File
@@ -1,253 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from .qwen3_moe import (
Qwen3MoeAttention,
Qwen3MoeDecoderLayer,
Qwen3MoeForCausalLM,
Qwen3MoeMLP,
Qwen3MoeModel,
Qwen3MoeSparseMoeBlock,
)
from .utils import PPMissingLayer, extract_layer_index, maybe_prefix
class MellumAttention(Qwen3MoeAttention):
"""Differences from `Qwen3MoeAttention`:
- Supports `per_layer_sliding_window` for `Attention`.
"""
def __init__(
self,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
rope_parameters: dict[str, Any],
max_position_embeddings: int = 8192,
head_dim: int | None = None,
rms_norm_eps: float = 1e-06,
qkv_bias: bool = False,
cache_config: Any | None = None,
quant_config: Any | None = None,
prefix: str = "",
dual_chunk_attention_config: dict[str, Any] | None = None,
per_layer_sliding_window: int | None = None,
) -> None:
nn.Module.__init__(self)
self.hidden_size = hidden_size
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = head_dim or (hidden_size // self.total_num_heads)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self.max_position_embeddings = max_position_embeddings
self.dual_chunk_attention_config = dual_chunk_attention_config
self.qkv_proj = QKVParallelLinear(
hidden_size,
self.head_dim,
self.total_num_heads,
self.total_num_kv_heads,
bias=qkv_bias,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
if dual_chunk_attention_config:
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=per_layer_sliding_window,
prefix=f"{prefix}.attn",
layer_idx=extract_layer_index(prefix),
dual_chunk_attention_config=dual_chunk_attention_config,
)
else:
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=per_layer_sliding_window,
prefix=f"{prefix}.attn",
)
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
class MellumDecoderLayer(Qwen3MoeDecoderLayer):
"""Differences from `Qwen3MoeDecoderLayer`:
- Supports interleaved SWA and per-layer RoPE scaling.
"""
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
nn.Module.__init__(self)
config = vllm_config.model_config.hf_text_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.hidden_size = config.hidden_size
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
dual_chunk_attention_config = getattr(
config, "dual_chunk_attention_config", None
)
layer_idx = extract_layer_index(prefix)
layer_type = config.layer_types[layer_idx]
if layer_type == "sliding_attention":
sliding_window = getattr(config, "sliding_window", None)
else:
sliding_window = None
rope_parameters = config.rope_parameters[layer_type]
self.self_attn = MellumAttention(
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=config.num_key_value_heads,
rope_parameters=rope_parameters,
max_position_embeddings=max_position_embeddings,
rms_norm_eps=config.rms_norm_eps,
qkv_bias=getattr(config, "attention_bias", False),
head_dim=getattr(config, "head_dim", None),
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
dual_chunk_attention_config=dual_chunk_attention_config,
per_layer_sliding_window=sliding_window,
)
if config.mlp_layer_types[layer_idx] == "sparse":
self.mlp = Qwen3MoeSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
)
else:
self.mlp = Qwen3MoeMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
@support_torch_compile
class MellumModel(Qwen3MoeModel):
"""Differences from `Qwen3MoeModel`:
- Uses `MellumDecoderLayer`.
"""
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
):
super().__init__(
vllm_config=vllm_config,
prefix=prefix,
decoder_layer_type=MellumDecoderLayer,
)
class MellumForCausalLM(Qwen3MoeForCausalLM):
"""Differences from `Qwen3MoeForCausalLM`:
- Uses `MellumModel`.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
config = vllm_config.model_config.hf_text_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
if "dense" in getattr(config, "mlp_layer_types", []):
self.packed_modules_mapping["gate_up_proj"] = ["gate_proj", "up_proj"]
self.model = MellumModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if self.config.tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
self.moe_layers = []
example_layer = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
continue
assert isinstance(layer, Qwen3MoeDecoderLayer)
if isinstance(layer.mlp, Qwen3MoeSparseMoeBlock):
example_layer = layer.mlp
self.moe_layers.append(layer.mlp.experts)
if example_layer is None:
raise RuntimeError("No MoE layer found in the model.layers.")
self.num_moe_layers = len(self.moe_layers)
self.num_expert_groups = 1
self.num_shared_experts = 0
self.num_logical_experts = example_layer.n_logical_experts
self.num_physical_experts = example_layer.n_physical_experts
self.num_local_physical_experts = example_layer.n_local_physical_experts
self.num_routed_experts = example_layer.n_routed_experts
self.num_redundant_experts = example_layer.n_redundant_experts
+6 -2
View File
@@ -742,8 +742,12 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
expert_params_mapping = self.get_expert_mapping()
# Pro-format fused qkv_proj arrives as two tensors (weight and
# weight_scale_inv). Store them per-layer so that they can be
# sharded together.
pending_fp8_qkv_proj: dict[str, dict[str, torch.Tensor]] = {}
# sharded together. The state must outlive this call: AutoWeightsLoader
# delegates per contiguous group of names, so a pair can straddle two
# calls and would otherwise be dropped silently.
pending_fp8_qkv_proj = getattr(self, "_pending_fp8_qkv_proj", None)
if pending_fp8_qkv_proj is None:
self._pending_fp8_qkv_proj = pending_fp8_qkv_proj = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
+6 -1
View File
@@ -266,7 +266,12 @@ class MiMoV2MTP(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
pending_qkv_proj: dict[str, dict[str, torch.Tensor]] = {}
# The pairing state must outlive this call: AutoWeightsLoader delegates
# per contiguous group of names, so a pair can straddle two calls and
# would otherwise be dropped silently.
pending_qkv_proj = getattr(self, "_pending_qkv_proj", None)
if pending_qkv_proj is None:
self._pending_qkv_proj = pending_qkv_proj = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
+20 -4
View File
@@ -69,7 +69,10 @@ from .qwen2_5_vl import (
Qwen2_5_VLVideoInputs,
Qwen2_5_VLVideoPixelInputs,
)
from .qwen2_vl import _create_qwen2vl_field_factory
from .qwen2_vl import (
Qwen2VLMultiModalDataParser,
_create_qwen2vl_field_factory,
)
from .utils import AutoWeightsLoader, IntermediateTensors, WeightsMapper, maybe_prefix
@@ -670,6 +673,14 @@ class MiMoVisionTransformer(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class MiMoV2OmniMultiModalDataParser(Qwen2VLMultiModalDataParser):
# Video stays raw: get_video_replacement also reads second_per_grid_ts and
# video_start_times, which a metadata-only rewrite would not carry.
embedding_fields = {
"image": Qwen2VLMultiModalDataParser.embedding_fields["image"],
}
class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None, "image": None, "video": None}
@@ -689,9 +700,14 @@ class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
return self.get_hf_processor(**kwargs).image_processor
def get_data_parser(self):
from vllm.multimodal.parse import MultiModalDataParser
return MultiModalDataParser(target_sr=24000.0)
# Without embedding_fields the EC producer publishes no metadata and
# the EPD proxy never rewrites the media item.
return MiMoV2OmniMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
target_sr=24000.0,
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_mm_max_tokens_per_item(
self,
+19 -3
View File
@@ -7,6 +7,8 @@ from typing import Annotated, Any, Literal, cast
import torch
import torch.nn as nn
import transformers
from packaging.version import Version
from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
from transformers.models.pixtral import PixtralProcessor
@@ -14,6 +16,7 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -56,6 +59,10 @@ from .utils import (
maybe_prefix,
)
TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version(
"5.15.0"
)
class Mistral3ImagePixelInputs(TensorSchema):
"""Dimensions:
@@ -225,7 +232,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
image_size = size["longest_edge"]
return Mistral3HFEncoderInfo(self.get_hf_config(), image_size)
def get_hf_processor(self, **kwargs: object):
def get_hf_processor(self, **kwargs: object) -> PixtralProcessor:
return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -277,8 +284,12 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
# PixtralProcessor supports image-only calls starting in transformers 5.15.
# Older releases need dummy text and take the slower tokenization path.
if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY:
return self.dummy_inputs.get_dummy_text(mm_counts)
return None
def _postprocess_hf_mm_data(
self,
@@ -378,6 +389,7 @@ def init_vision_tower_for_mistral3(
hf_config: Mistral3Config,
quant_config: QuantizationConfig | None,
*,
input_norm: nn.Module | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
) -> PixtralHFVisionModel:
@@ -391,6 +403,7 @@ def init_vision_tower_for_mistral3(
return PixtralHFVisionModel(
vision_config,
quant_config=quant_config,
input_norm=input_norm,
num_hidden_layers_override=num_hidden_layers,
require_post_norm=require_post_norm,
prefix=prefix,
@@ -410,6 +423,8 @@ class Mistral3ForConditionalGeneration(
SupportsEagle,
SupportsEagle3,
):
supports_mm_device_do_normalize = True
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -464,6 +479,7 @@ class Mistral3ForConditionalGeneration(
self.vision_tower = init_vision_tower_for_mistral3(
config,
quant_config=quant_config,
input_norm=build_mm_input_norm(vllm_config.model_config),
require_post_norm=False,
prefix=maybe_prefix(prefix, "vision_tower"),
)
+42 -13
View File
@@ -33,6 +33,7 @@ from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
@@ -77,23 +78,22 @@ def _cast_if_autocast_enabled(device_type: str, *args):
)
class NemotronLayerNorm1P(nn.LayerNorm):
def __init__(
self,
normalized_shape: int | list[int] | torch.Size,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
device=None,
dtype=None,
):
super().__init__(normalized_shape, eps, elementwise_affine, bias, device, dtype)
@CustomOp.register("nemotron_layer_norm")
class NemotronLayerNorm1P(CustomOp):
"""LayerNorm variant used by Nemotron: `x * (1 + w)` instead of `x * w`."""
def forward(
def __init__(self, hidden_size: int, eps: float = 1e-5) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
self.weight = nn.Parameter(torch.zeros(hidden_size))
self.bias = nn.Parameter(torch.zeros(hidden_size))
def forward_native(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if residual is not None:
x = x + residual
residual = x
@@ -105,6 +105,35 @@ class NemotronLayerNorm1P(nn.LayerNorm):
x = torch.nn.functional.layer_norm(*args)
return x if residual is None else (x, residual)
def forward_cuda(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return self.forward_native(x, residual)
def forward_xpu(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_nemotron_layer_norm
if not hasattr(torch.ops._C, "nemotron_layer_norm"):
return self.forward_native(x, residual)
if residual is not None:
if not hasattr(torch.ops._C, "fused_add_nemotron_layer_norm"):
return self.forward_native(x, residual)
torch.ops.vllm.xpu_fused_add_nemotron_layer_norm(
x, residual, self.weight, self.bias, self.eps
)
return x, residual
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_nemotron_layer_norm(out, x, self.weight, self.bias, self.eps)
return out
class NemotronMLP(nn.Module):
def __init__(
+1
View File
@@ -815,6 +815,7 @@ class NemotronHForCausalLM(
tp_world_size=parallel_config.tensor_parallel_size,
logical_window=cache_config.replayssm_buffer_len,
backend=vllm_config.mamba_config.backend,
num_speculative_tokens=vllm_config.num_speculative_tokens,
)
return base_shape
@@ -568,6 +568,15 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
self.lm_head = ParallelLMHead(
config.decoder.vocab_size, config.decoder.d_model, quant_config=quant_config
)
# Some checkpoints (e.g. compact exports) tie the output head to the
# decoder's input embeddings instead of materializing a separate
# lm_head.weight tensor.
tie_word_embeddings = bool(
getattr(config, "tie_word_embeddings", False)
or getattr(config.decoder, "tie_word_embeddings", False)
)
if tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.decoder.embed_tokens)
self.logits_processor = LogitsProcessor(
self.vocab_size, config.decoder.vocab_size
)
+20
View File
@@ -8,6 +8,10 @@ import torch.nn as nn
from transformers import AutoModel, PreTrainedConfig
from vllm.config import VllmConfig
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
)
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -26,6 +30,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_image_processor_from_config
from vllm.transformers_utils.processors.nemotron_vl import (
SIGLIP_MEAN,
SIGLIP_STD,
LlamaNemotronNanoVLImageProcessor,
LlamaNemotronNanoVLProcessor,
LlamaNemotronVLEmbedImageProcessor,
@@ -474,6 +480,7 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
"""
is_pooling_model = True
supports_mm_device_do_normalize = True
# Weight mapping from checkpoint format to vLLM format
# Different from parent class due to different vision model structure
@@ -498,6 +505,15 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
# Override: get img_context_token_id from config (parent sets None)
self.img_context_token_id = getattr(config, "img_context_token_id", None)
if self.model_config.get_multimodal_config().mm_device_do_normalize:
self.input_norm = FusedMMInputNorm(
image_mean=list(SIGLIP_MEAN),
image_std=list(SIGLIP_STD),
rescale_factor=1 / 255,
)
else:
self.input_norm = IdentityInputNorm()
# Initialize pooler for embedding output
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
@@ -537,6 +553,10 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
def _call_vision_model(self, pixel_values: torch.Tensor) -> torch.Tensor:
"""Override to handle SigLIP interface."""
original_shape = pixel_values.shape
pixel_values = self.input_norm(
pixel_values.flatten(start_dim=1), self.vision_model.dtype
).view(original_shape)
return self.vision_model(pixel_values)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+5 -3
View File
@@ -32,6 +32,7 @@ from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -143,7 +144,7 @@ class OPTDecoderLayer(nn.Module):
)
self.do_layer_norm_before = config.do_layer_norm_before
self.self_attn_layer_norm = nn.LayerNorm(
self.self_attn_layer_norm = LayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)
self.fc1 = ColumnParallelLinear(
@@ -161,7 +162,7 @@ class OPTDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.fc2",
)
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = LayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)
@@ -246,8 +247,9 @@ class OPTDecoder(nn.Module):
# keep backward compatibility with checkpoints that have been fine-tuned
# before transformers v4.20.1
# see https://github.com/facebookresearch/metaseq/pull/164
self.final_layer_norm: LayerNorm | None
if config.do_layer_norm_before and not config._remove_final_layer_norm:
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = LayerNorm(
config.hidden_size,
elementwise_affine=config.layer_norm_elementwise_affine,
)
+95 -14
View File
@@ -10,6 +10,7 @@ from typing import Annotated, Literal
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
@@ -20,6 +21,10 @@ from transformers.models.pixtral.image_processing_pixtral import (
)
from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb
from vllm.compilation.decorators import (
should_torch_compile_mm_encoder,
support_torch_compile,
)
from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -828,6 +833,16 @@ class Attention(nn.Module):
return out
@support_torch_compile(
dynamic_arg_dims={
"x": 1,
"freqs_cis": 0,
"cu_seqlens": 0,
"sequence_lengths": 0,
},
enable_if=should_torch_compile_mm_encoder,
is_encoder=True,
)
class TransformerBlock(nn.Module):
def __init__(
self,
@@ -1297,7 +1312,7 @@ class PixtralHFAttention(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
@@ -1328,6 +1343,17 @@ class PixtralHFAttention(nn.Module):
return attn_output, None
@support_torch_compile(
dynamic_arg_dims={
"hidden_states": 1,
"rotary_pos_emb_cos": 0,
"rotary_pos_emb_sin": 0,
"cu_seqlens": 0,
"sequence_lengths": 0,
},
enable_if=should_torch_compile_mm_encoder,
is_encoder=True,
)
class PixtralHFTransformerBlock(nn.Module):
def __init__(
self,
@@ -1354,14 +1380,15 @@ class PixtralHFTransformerBlock(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: torch.Tensor,
rotary_pos_emb_cos: torch.Tensor,
rotary_pos_emb_sin: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
) -> torch.Tensor:
r, _ = self.attention.forward(
self.attention_norm(hidden_states),
position_embeddings=position_embeddings,
position_embeddings=(rotary_pos_emb_cos, rotary_pos_emb_sin),
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
@@ -1402,18 +1429,20 @@ class PixtralHFTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
position_embeddings: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
return_all_hidden_states: bool,
) -> torch.Tensor:
hidden_states_pool = [x]
rotary_pos_emb_cos, rotary_pos_emb_sin = position_embeddings
for layer in self.layers:
x = layer(
x,
position_embeddings,
rotary_pos_emb_cos,
rotary_pos_emb_sin,
cu_seqlens,
max_seqlen,
sequence_lengths,
@@ -1427,12 +1456,55 @@ class PixtralHFTransformer(nn.Module):
return x
def pixtral_patch_embed(
images: list[torch.Tensor],
weight: torch.Tensor,
input_norm: nn.Module,
) -> tuple[torch.Tensor, list[torch.Tensor]]:
"""Normalize and project all CHW images in one matrix multiplication."""
out_channels, in_channels, patch_height, patch_width = weight.shape
grid_sizes = []
for image in images:
assert image.ndim == 3 and image.shape[0] == in_channels
_, height, width = image.shape
patch_rows = height // patch_height
patch_cols = width // patch_width
grid_sizes.append((patch_rows, patch_cols))
patch_counts = [rows * cols for rows, cols in grid_sizes]
packed = torch.empty(
(sum(patch_counts), in_channels * patch_height * patch_width),
dtype=images[0].dtype,
device=images[0].device,
)
offset = 0
for image, (rows, cols), count in zip(images, grid_sizes, patch_counts):
source = (
image[:, : rows * patch_height, : cols * patch_width]
.reshape(in_channels, rows, patch_height, cols, patch_width)
.permute(1, 3, 0, 2, 4)
)
packed[offset : offset + count].view(
rows, cols, in_channels, patch_height, patch_width
).copy_(source)
offset += count
normalized = input_norm(packed, weight.dtype)
projected = F.linear(normalized, weight.reshape(out_channels, -1))
embeddings = [
tokens.transpose(0, 1).reshape(1, out_channels, rows, cols)
for tokens, (rows, cols) in zip(projected.split(patch_counts), grid_sizes)
]
return projected.unsqueeze(0), embeddings
class PixtralHFVisionModel(nn.Module):
def __init__(
self,
config: PixtralVisionConfig,
quant_config: QuantizationConfig | None = None,
*,
input_norm: nn.Module | None = None,
num_hidden_layers_override: int | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
@@ -1440,6 +1512,7 @@ class PixtralHFVisionModel(nn.Module):
super().__init__()
self.config = config
self.input_norm = input_norm
self.patch_conv = Conv2dLayer(
in_channels=config.num_channels,
@@ -1495,16 +1568,24 @@ class PixtralHFVisionModel(nn.Module):
all tokens of all images of shape (N_toks, D)
"""
# pass images through initial convolution independently
patch_embeds_list = [
self.patch_conv(img.unsqueeze(0).to(self.dtype)) for img in pixel_values
]
images = [image.to(device=self.device) for image in pixel_values]
if self.input_norm is not None:
patch_embeds, patch_embeds_list = pixtral_patch_embed(
images,
self.patch_conv.weight,
self.input_norm,
)
else:
patch_embeds_list = []
for image in images:
patch_embeds_list.append(
self.patch_conv(image.unsqueeze(0).to(self.dtype))
)
patch_embeds = torch.cat(
[p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1
)
patch_embeds = [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list]
embed_sizes = [p.shape[1] for p in patch_embeds]
# flatten to a single sequence
patch_embeds = torch.cat(patch_embeds, dim=1)
embed_sizes = [p.shape[-2] * p.shape[-1] for p in patch_embeds_list]
patch_embeds = self.ln_pre(patch_embeds)
# positional embeddings
-6
View File
@@ -331,14 +331,8 @@ class Qwen3_5ForCausalLMBase(
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
scheduler_config = vllm_config.scheduler_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3.5 currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
@@ -303,13 +303,6 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3_5MTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
-6
View File
@@ -828,14 +828,8 @@ class Qwen3NextForCausalLM(
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
scheduler_config = vllm_config.scheduler_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3Next currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
@@ -187,13 +187,6 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3NextMTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
+4 -5
View File
@@ -72,7 +72,6 @@ _TEXT_GENERATION_MODELS = {
# [Decoder-only]
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
"ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
"BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
@@ -84,7 +83,6 @@ _TEXT_GENERATION_MODELS = {
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
"Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
"Cohere2MoeForCausalLM": ("cohere2_moe", "Cohere2MoeForCausalLM"),
"CwmForCausalLM": ("llama", "LlamaForCausalLM"),
"DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
"DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
"DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
@@ -114,7 +112,6 @@ _TEXT_GENERATION_MODELS = {
"vllm.models.qwen4_exp",
"Qwen4ExpForCausalLM",
),
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
@@ -160,7 +157,6 @@ _TEXT_GENERATION_MODELS = {
),
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
"Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
"MellumForCausalLM": ("mellum", "MellumForCausalLM"),
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
"MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
"MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
@@ -226,7 +222,6 @@ _EMBEDDING_MODELS = {
"DeepseekV3BidirectionalModel": ("deepseek_v2", "DeepseekV3ForCausalLM"),
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
"Gemma3TextModel": ("gemma3", "Gemma3Model"),
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
"GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
"GteNewModel": ("bert_with_rope", "GteNewModel"),
"JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"),
@@ -709,10 +704,14 @@ _SPECULATIVE_DECODING_MODELS = {
_TRANSFORMERS_SUPPORTED_MODELS = {
# Text generation models
"ArceeForCausalLM": ("transformers", "TransformersForCausalLM"),
"CwmForCausalLM": ("transformers", "TransformersForCausalLM"),
"FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"GlmForCausalLM": ("transformers", "TransformersForCausalLM"),
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
+8 -3
View File
@@ -117,6 +117,8 @@ class RobertaClassificationHead(nn.Module):
class RobertaEmbeddingModel(BertEmbeddingModel):
"""A model that uses Roberta to provide embedding functionalities."""
embedding_class: type[nn.Module] = RobertaEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.padding_idx: int = vllm_config.model_config.hf_config.pad_token_id
@@ -143,7 +145,7 @@ class RobertaEmbeddingModel(BertEmbeddingModel):
return BertModel(
vllm_config=vllm_config,
prefix=prefix,
embedding_class=RobertaEmbedding,
embedding_class=self.embedding_class,
)
else:
return JinaRobertaModel(vllm_config=vllm_config, prefix=prefix)
@@ -293,6 +295,8 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, Support
}
)
embedding_class: type[nn.Module] = RobertaEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
@@ -303,7 +307,7 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, Support
self.roberta = BertModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "bert"),
embedding_class=RobertaEmbedding,
embedding_class=self.embedding_class,
)
self.classifier = RobertaClassificationHead(vllm_config.model_config)
@@ -355,6 +359,7 @@ class RobertaForTokenClassification(nn.Module):
"""
is_pooling_model = True
embedding_class: type[nn.Module] = RobertaEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -364,7 +369,7 @@ class RobertaForTokenClassification(nn.Module):
self.roberta = BertModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "roberta"),
embedding_class=RobertaEmbedding,
embedding_class=self.embedding_class,
)
self.classifier = nn.Linear(
config.hidden_size, config.num_labels, dtype=self.head_dtype
+23 -13
View File
@@ -73,6 +73,7 @@ from .utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
@@ -460,7 +461,9 @@ class SarvamMLABlock(nn.Module):
}
)
class SarvamMLAModel(nn.Module, EagleModelMixin):
"""Sarvam MLA backbone with stage-local EAGLE3 auxiliary capture."""
"""Sarvam MLA backbone with EAGLE3 auxiliary capture across pipeline stages."""
supports_aux_hidden_states_over_pp = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
@@ -487,8 +490,10 @@ class SarvamMLAModel(nn.Module, EagleModelMixin):
self.vocab_size = config.vocab_size
self.embed_dim = config.hidden_size
self.tie_word_embeddings = getattr(config, "tie_word_embeddings", False)
if get_pp_group().is_first_rank or (
self.tie_word_embeddings and get_pp_group().is_last_rank
if (
get_pp_group().is_first_rank
or (self.tie_word_embeddings and get_pp_group().is_last_rank)
or spec_decode_needs_target_embed(vllm_config)
):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
@@ -530,13 +535,10 @@ class SarvamMLAModel(nn.Module, EagleModelMixin):
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
"""Run this stage and optionally return its auxiliary hidden states.
Auxiliary captures are local to this stage, matching Qwen3 MoE.
Pipeline transport carries only hidden states and residual; EAGLE3
capture across pipeline stages is not supported by this model.
Returns:
Intermediate tensors on non-final stages. On the final stage,
normalized hidden states, paired with auxiliary states if captured.
Intermediate tensors with local auxiliary states on non-final stages.
On the final stage, normalized hidden states paired with the ordered
auxiliary states from all stages if captured.
"""
if get_pp_group().is_first_rank:
@@ -551,9 +553,12 @@ class SarvamMLAModel(nn.Module, EagleModelMixin):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state(
[], self.start_layer, hidden_states, residual
)
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
if get_pp_group().is_first_rank:
self._maybe_add_hidden_state(
aux_hidden_states, self.start_layer, hidden_states, residual
)
for layer_idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
@@ -569,13 +574,18 @@ class SarvamMLAModel(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
+7 -5
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -467,13 +468,13 @@ class SiglipEncoderLayer(nn.Module):
prefix=f"{prefix}.self_attn",
attn_cls=attn_cls,
)
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.layer_norm1 = LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.mlp = SiglipMLP(
config,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.layer_norm2 = LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
def forward(
self,
@@ -573,7 +574,7 @@ class SiglipTextTransformer(nn.Module):
attn_cls=EncoderOnlyAttention,
)
self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.final_layer_norm = LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.head = nn.Linear(embed_dim, config.projection_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -616,7 +617,7 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
self.attention = torch.nn.MultiheadAttention(
config.hidden_size, config.num_attention_heads, batch_first=True
)
self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
self.layernorm = LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
self.mlp = SiglipMLP(
config=config,
quant_config=quant_config,
@@ -685,8 +686,9 @@ class SiglipVisionTransformer(nn.Module):
if require_post_norm is None:
require_post_norm = len(self.encoder.layers) == num_hidden_layers
self.post_layernorm: LayerNorm | None
if require_post_norm:
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.post_layernorm = LayerNorm(embed_dim, eps=config.layer_norm_eps)
else:
self.post_layernorm = None
+16 -1
View File
@@ -22,7 +22,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .step3p5 import Step3p5DecoderLayer
from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix
from .utils import (
get_draft_quant_config,
get_spec_layer_idx_from_weight_name,
maybe_prefix,
)
logger = init_logger(__name__)
@@ -152,8 +156,19 @@ class Step3p5AMultiTokenPredictor(nn.Module):
class Step3p5MTP(nn.Module):
# Each MTP layer's shared_head carries its own LM head weights.
has_own_lm_head = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
# The target config may be a multimodal wrapper (Step3.7); build the
# draft from its own text and quant configs.
assert vllm_config.speculative_config is not None
draft_quant_config = get_draft_quant_config(vllm_config)
vllm_config = vllm_config.with_hf_config(
vllm_config.speculative_config.draft_model_config.hf_config
)
vllm_config.quant_config = draft_quant_config
self.config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config
self.model = Step3p5AMultiTokenPredictor(
@@ -71,6 +71,7 @@ from vllm.model_executor.models.transformers.utils import (
named_state,
replace_conv_class,
replace_embedding_class,
replace_layernorm_class,
replace_linear_class,
)
from vllm.model_executor.models.utils import (
@@ -466,6 +467,7 @@ class Base(
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
- `nn.LayerNorm` with vLLM's `LayerNorm`
"""
tp_plan = self.model.tp_plan or {}
@@ -556,6 +558,8 @@ class Base(
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
elif isinstance(child_module, nn.LayerNorm):
new_module = replace_layernorm_class(child_module)
elif child_module_fusers := fusers[child_module]:
for fuser in child_module_fusers:
register_fusion(fuser, qual_name, child_module)
@@ -28,6 +28,7 @@ from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
@@ -187,6 +188,34 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
)
def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module:
"""Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's
`LayerNorm`.
Args:
layernorm: `nn.LayerNorm` to be replaced.
Returns:
The new `LayerNorm`. If the layernorm is not supported (a subclass with
its own behavior, multi-dim `normalized_shape`, or not both
elementwise-affine and biased), returns the original module unchanged.
"""
if (
type(layernorm) is not nn.LayerNorm
or len(layernorm.normalized_shape) != 1
or not layernorm.elementwise_affine
or layernorm.bias is None
):
return layernorm
return LayerNorm(
layernorm.normalized_shape[0],
eps=layernorm.eps,
dtype=layernorm.weight.dtype,
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")
+19 -4
View File
@@ -321,6 +321,14 @@ class AutoWeightsLoader:
f"into a single parameter {base_prefix!r}"
)
if param.dtype != weight_data.dtype:
logger.warning(
"Attempted to load weight %s with dtype %s into "
"parameter with dtype %s",
weight_qualname,
weight_data.dtype,
param.dtype,
)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, weight_data)
@@ -844,17 +852,24 @@ class PPMissingLayer(torch.nn.Identity):
return args[0] if args else next(iter(kwargs.values()))
def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool:
"""Whether the last PP rank needs the target input embedding."""
def spec_decode_needs_target_embed(
vllm_config: VllmConfig, *, include_mtp: bool = False
) -> bool:
"""Whether the last PP rank needs the target input embedding.
MTP targets opt in when their drafter supports sharing this embedding.
"""
from vllm.distributed.parallel_state import get_pp_group
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.method not in (
if speculative_config is None:
return False
if speculative_config.method not in (
"eagle",
"eagle3",
"dflash",
"dspark",
):
) and not (include_mtp and speculative_config.method == "mtp"):
return False
pp = get_pp_group()
return pp.world_size > 1 and pp.is_last_rank
+52 -12
View File
@@ -63,6 +63,7 @@ from vllm.multimodal.processing import (
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.renderers import TokenizeParams
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.jsontree import json_map_leaves
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -325,7 +326,7 @@ class WhisperCrossAttention(WhisperAttention):
prefix=f"{prefix}.kv_proj",
)
def forward(
def forward( # type: ignore[override]
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor | None,
@@ -872,7 +873,7 @@ class WhisperForConditionalGeneration(
@classmethod
def get_language_token_ids(
cls,
tokenizer: object,
tokenizer: TokenizerLike,
) -> list[int]:
"""Return token IDs for all supported language tokens.
@@ -908,8 +909,8 @@ class WhisperForConditionalGeneration(
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: object,
) -> str | None:
tokenizer: TokenizerLike,
) -> str:
"""Parse the language token predicted by Whisper.
Decodes the first token ID and extracts the language code from the
@@ -1004,6 +1005,7 @@ class WhisperForConditionalGeneration(
audio_input = self._parse_and_validate_audio_input(**kwargs)
# Split concatenated encoder outputs into one tensor per audio input
enc_output = self.model.get_encoder_outputs(audio_input["input_features"])
assert enc_output is not None
# The assumption is we can only process whole mm items (audios)
return enc_output.unbind(dim=0)
@@ -1021,8 +1023,12 @@ class WhisperForConditionalGeneration(
def _parse_and_validate_audio_input(self, **kwargs: object) -> WhisperAudioInputs:
input_features = kwargs.pop("input_features", None)
def to_dtype(value: object) -> torch.Tensor:
assert isinstance(value, torch.Tensor)
return value.to(self.dtype)
if input_features is not None:
input_features = json_map_leaves(lambda x: x.to(self.dtype), input_features)
input_features = json_map_leaves(to_dtype, input_features)
return WhisperAudioInputs(input_features=input_features)
@@ -1033,20 +1039,54 @@ class WhisperForConditionalGeneration(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
# add fake zeros bias for k_proj to state_dict
weights = _create_fake_bias_for_k_proj(weights, ".k_proj.weight")
# add fake zeros bias for k_proj to state_dict. Covers the
# ``.k_proj.weight`` and the compressed-tensors packed
# ``.k_proj.weight_packed`` checkpoints.
k_proj_weight_suffixes = (".k_proj.weight", ".k_proj.weight_packed")
for suffix in k_proj_weight_suffixes:
weights = _create_fake_bias_for_k_proj(
weights, suffix, out_features=self.config.d_model
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def _create_fake_bias_for_k_proj(
weights: Iterable[tuple[str, torch.Tensor]], fake_bias_key_name: str
weights: Iterable[tuple[str, torch.Tensor]],
fake_bias_key_name: str,
out_features: int | None = None,
) -> Iterable[tuple[str, torch.Tensor]]:
"""Create full zeros bias for k_proj weight in self-attn and x-attn layers.
So that the bias for k_proj in qkv_proj can be initialized with zeros.
So that the bias for k_proj in qkv_proj or kv_proj can be initialized with
zeros.
If the checkpoint already provides a real ``.bias`` entry for the given
weight, it is forwarded as-is and no fake bias is injected for that layer.
"""
# Map the weight-name suffix to the corresponding bias-name suffix, e.g.
# ".k_proj.weight_packed" / ".k_proj.weight" -> ".k_proj.bias",
# ".wk.weight" -> ".wk.bias". ``.weight_packed`` is stripped first so the
# longer suffix wins.
bias_key_name = (
fake_bias_key_name.removesuffix(".weight_packed").removesuffix(".weight")
+ ".bias"
)
real_bias_names: set[str] = set()
pending: dict[str, torch.Tensor] = {}
for name, weight in weights:
yield name, weight
if name.endswith(bias_key_name):
real_bias_names.add(name)
pending.pop(name, None)
continue
if name.endswith(fake_bias_key_name):
bias = torch.zeros(weight.size(0))
bias_name = name.replace("weight", "bias")
yield bias_name, bias
bias_name = name[: -len(fake_bias_key_name)] + bias_key_name
if bias_name not in real_bias_names:
pending[bias_name] = torch.zeros(
out_features if out_features is not None else weight.size(0)
)
for bias_name, bias in pending.items():
yield bias_name, bias
+16 -8
View File
@@ -6,7 +6,7 @@ import math
from dataclasses import replace
from fractions import Fraction
from functools import partial
from typing import ClassVar
from typing import ClassVar, Protocol
import torch
import torch.nn.functional as F
@@ -35,11 +35,6 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.cpu_attn import CPUAttentionBackend
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
try:
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
except ImportError:
AiterFlashAttentionBackend = None
from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
from vllm.v1.attention.selector import get_attn_backend
@@ -47,11 +42,24 @@ from vllm.v1.kv_cache_interface import AttentionSpec, SlidingWindowSpec
from .utils import make_layers
AiterFlashAttentionBackend: type[AttentionBackend] | None = None
try:
from vllm.v1.attention.backends import rocm_aiter_fa
AiterFlashAttentionBackend = rocm_aiter_fa.AiterFlashAttentionBackend
except ImportError:
pass
logger = logging.getLogger(__name__)
CausalRMSNorm = partial(RMSNorm, eps=1e-5)
class _BlockPoolingAttentionMetadata(Protocol):
@property
def slot_mapping(self) -> torch.Tensor: ...
def _pad1d(
x: torch.Tensor,
paddings: tuple[int, int],
@@ -115,7 +123,7 @@ class WhisperCausalConv1d(nn.Conv1d):
@functools.lru_cache
def create_whisper_attention_backend_with_block_pooling(
underlying_attn_backend: AttentionBackend,
underlying_attn_backend: type[AttentionBackend],
block_pool_size: int,
sliding_window: int | None = None,
) -> type[AttentionBackend]:
@@ -270,7 +278,7 @@ def create_whisper_attention_backend_with_block_pooling(
key: torch.Tensor,
value: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata: AttentionMetadata,
attn_metadata: _BlockPoolingAttentionMetadata | None,
output: torch.Tensor,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
+15 -10
View File
@@ -86,16 +86,21 @@ class Zamba2LoRA(nn.Module):
)
if isinstance(output_dim, list):
B_class = MergedColumnParallelLinear
self.B: ColumnParallelLinear = MergedColumnParallelLinear(
rank,
output_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.B",
)
else:
B_class = ColumnParallelLinear
self.B = B_class(
rank,
output_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.B",
)
self.B = ColumnParallelLinear(
rank,
output_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.B",
)
def forward(
self,
@@ -304,7 +309,7 @@ class Zamba2MLP(nn.Module):
self,
config: Zamba2Config,
bare_block_idx: int,
num_hybrid_layers: dict[int, int],
num_hybrid_layers: int,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
+21 -9
View File
@@ -410,17 +410,24 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
Without autotuning, FlashInfer will rely on heuristics, which may
be significantly slower.
Every rank profiles the same tactics. When distributed, per-tactic
timings are averaged over the world CPU group so all ranks select the
same tactic.
With PP > 1, stages run different layers and may profile different ops,
so each stage's TP group tunes separately with its own cache file;
otherwise the world group tunes together. Per-tactic timings are
averaged over the tuning group so all its ranks select the same tactic.
"""
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
import vllm.utils.flashinfer as fi_utils
from vllm.distributed.parallel_state import get_world_group
from vllm.distributed.parallel_state import (
get_pp_group,
get_tp_group,
get_world_group,
)
world = get_world_group()
is_leader = world.rank_in_group == 0
pp_size = get_pp_group().world_size
tune_group = get_tp_group() if pp_size > 1 else world
is_leader = tune_group.rank_in_group == 0
tuner = AutoTuner.get()
autotune_kwargs: dict = {}
@@ -433,6 +440,11 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
autotune_kwargs["skip_ops"] = skip_ops
cache_path = resolve_flashinfer_autotune_file(runner)
if pp_size > 1:
ranks = "-".join(str(rank) for rank in tune_group.ranks)
cache_path = cache_path.with_name(
f"{cache_path.stem}_tp_{ranks}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
@@ -441,18 +453,18 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
# Read cached autotune results and broadcast to all ranks.
# Read cached autotune results and broadcast within the tuning group.
cached_results: bytes | None = None
if is_leader and cache_path.exists():
with open(cache_path, "rb") as f:
cached_results = f.read()
cached_results = world.broadcast_object(cached_results, src=0)
cached_results = tune_group.broadcast_object(cached_results, src=0)
if cached_results is not None:
write_flashinfer_autotune_cache(cache_path, cached_results)
world.barrier()
tune_group.barrier()
tuner.load_configs(str(cache_path))
group = world.cpu_group if world.world_size > 1 else None
group = tune_group.cpu_group if tune_group.world_size > 1 else None
set_autotune_process_group(group)
try:
with (
@@ -4,6 +4,7 @@
from __future__ import annotations
import sys
from typing import TYPE_CHECKING
import torch
@@ -19,7 +20,10 @@ logger = init_logger(__name__)
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
# Kimi model construction already imports kda. Avoid importing it here.
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
if kda is None:
return None
compilation_config = getattr(
worker.model_runner,
@@ -33,7 +37,7 @@ def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
(
layer
for layer in static_context.values()
if isinstance(layer, KimiK3DeltaAttention)
if isinstance(layer, kda.KimiK3DeltaAttention)
),
None,
)
+30 -7
View File
@@ -21,6 +21,7 @@ logger = init_logger(__name__)
def _replayssm_autotune_kwargs(
runner: "GPUModelRunner",
max_token_prefill_kwargs: dict[str, Any],
) -> tuple[int, dict[str, Any]] | None:
config = runner.vllm_config
if not (
@@ -46,11 +47,16 @@ def _replayssm_autotune_kwargs(
runner.max_num_tokens // query_len,
runner.kv_cache_config.num_blocks - 1,
)
if max_num_reqs <= 0:
logger.warning_once(
"Skipping FlashInfer ReplaySSM autotuning because no non-padding "
"state slot is available."
)
return None
decode_kwargs = {
**max_token_prefill_kwargs,
"num_tokens": max_num_reqs * query_len,
"skip_eplb": True,
"is_profile": True,
"randomize_inputs": True,
"uniform_decode": True,
}
if config.use_v2_model_runner:
@@ -75,18 +81,22 @@ def _temporary_replayssm_autotune_state(
)
reset_tensors: dict[int, torch.Tensor] = {}
tracker_specs: dict[int, tuple[torch.Tensor, torch.Tensor, int, int]] = {}
tracker_specs: dict[
int, tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]
] = {}
for module in runner.get_model().modules():
if not isinstance(module, MambaMixer2) or not module.use_replayssm:
continue
assert module.replayssm_buffer_len is not None
ring_start = module._replayssm_ring_start
prev_num_accepted = module._replayssm_prev_num_accepted
prev_query_len = module._replayssm_prev_query_len
tracker_specs.setdefault(
ring_start.data_ptr(),
(
ring_start,
prev_num_accepted,
prev_query_len,
module.replayssm_buffer_len,
module.kv_cache[2].size(2),
),
@@ -95,6 +105,7 @@ def _temporary_replayssm_autotune_state(
*module.kv_cache,
ring_start,
prev_num_accepted,
prev_query_len,
)
for tensor in tensors:
if tensor.numel():
@@ -121,20 +132,26 @@ def _temporary_replayssm_autotune_state(
for (
ring_start,
prev_num_accepted,
prev_query_len,
logical_window,
ring_buffer_len,
) in tracker_specs.values():
# Compile reset (prefill) and advance (decode) before inference.
# The final reset leaves the decode tuning run in a clean state.
reset_replayssm_ring_trackers(ring_start, prev_num_accepted, state_slots)
reset_replayssm_ring_trackers(
ring_start, prev_num_accepted, prev_query_len, state_slots
)
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_slots,
logical_window,
ring_buffer_len,
)
reset_replayssm_ring_trackers(ring_start, prev_num_accepted, state_slots)
reset_replayssm_ring_trackers(
ring_start, prev_num_accepted, prev_query_len, state_slots
)
try:
yield
@@ -151,7 +168,13 @@ def _temporary_replayssm_autotune_state(
def replayssm_autotune_warmup(runner: "GPUModelRunner") -> None:
autotune = _replayssm_autotune_kwargs(runner)
max_token_prefill_kwargs = {
"num_tokens": runner.scheduler_config.max_num_batched_tokens,
"skip_eplb": True,
"is_profile": True,
"randomize_inputs": True,
}
autotune = _replayssm_autotune_kwargs(runner, max_token_prefill_kwargs)
if autotune is None:
return
max_num_reqs, decode_kwargs = autotune