Merge branch 'main' into qwen4exp-qsa-fused-draft-metadata

This commit is contained in:
vr8vr8
2026-09-23 23:37:13 +03:00
committed by GitHub
74 changed files with 3474 additions and 387 deletions
+85
View File
@@ -1818,6 +1818,48 @@ steps:
commands:
- pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py
- label: ":amd: (MI355) Kimi K3" # TBD
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_1
num_gpus: 1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/models/kimi_k3/
- csrc/libtorch_stable/kimi_k3/
- tests/models/kimi_k3/
- vllm/platforms/rocm.py
commands:
# Mirrors the ":amd: (MI355) Kimi K3" step generated from
# .buildkite/test_areas/models_basic.yaml. The ROCm KDA/AttnRes kernels are
# gated on gfx950, so this has to be MI355. The two kernels/ files the NVIDIA
# step runs are CUDA-only (they import cuda.bindings.driver), so they are
# deliberately left out here.
- pytest -v -s models/kimi_k3
- label: ":amd: (MI355) Kimi K3 Multi-GPU" # TBD
timeout_in_minutes: 30
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/models/kimi_k3/
- tests/models/kimi_k3/
- vllm/platforms/rocm.py
commands:
# The single-GPU step above skips every @multi_gpu_test case. `-m "distributed"`
# selects exactly those: multi_gpu_test() applies pytest.mark.distributed
# alongside its skipif (tests/utils.py:2080), so this picks up a new
# multi-GPU test anywhere in the directory without editing this list.
# Four GPUs run the two tp4 cases in test_amd_latent_moe_runner.py; the tp8
# case and the 8/16-GPU cases in test_latent_moe_tail.py still skip.
- pytest -v -s -m "distributed" models/kimi_k3
- label: ":amd: (MI300) Mamba Kernels" # TBD
timeout_in_minutes: 55
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -3562,6 +3604,49 @@ steps:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
- label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
timeout_in_minutes: 30
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
- tests/v1/kv_connector/mooncake_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- examples/disaggregated/mooncake_connector/
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
- label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" # TBD
timeout_in_minutes: 40
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+23
View File
@@ -42,6 +42,29 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# Preserve all six model/TP pairs with native AITER attention.
- pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
- label: ":nvidia: (L4) Push NixlConnector PP prefill PD accuracy"
key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
@@ -19,6 +19,30 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
- tests/v1/kv_connector/mooncake_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- examples/disaggregated/mooncake_connector/
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# Same full accuracy/TP sweep, using same-host HIP GPU transport.
- pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
- label: ":nvidia: (L4) Mooncake EC TCP E2E"
key: mooncake-ec-tcp-e2e-2-gpus
+1
View File
@@ -12,6 +12,7 @@ steps:
working_dir: "."
env:
SNAPSHOT_E2E_RUN_LIMIT_S: "1200"
NVIDIA_VISIBLE_DEVICES: "0"
source_file_dependencies:
- .buildkite/scripts/initialized-snapshot-e2e.sh
- .buildkite/test_areas/entrypoints.yaml
+2 -2
View File
@@ -571,7 +571,7 @@ steps:
- label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY"
key: lm-eval-humming-h100-act
optional: true
timeout_in_minutes: 70
timeout_in_minutes: 90
device: h100
num_devices: 1
source_file_dependencies:
@@ -582,7 +582,7 @@ steps:
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
# - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: ":nvidia: (H100) LM Eval Humming DeepEP TEMPORARY"
key: lm-eval-humming-h100-deepep
+17
View File
@@ -109,6 +109,23 @@ steps:
commands:
# The native NVIDIA Kimi K3 kernels require the SM100 family.
- pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py
mirror:
amd:
label: ":amd: (MI355) Kimi K3"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
commands:
# The ROCm KDA/AttnRes kernels are gated on gfx950, so this has to be MI355.
# The two kernels/ files the NVIDIA job adds are CUDA-only (they import
# cuda.bindings.driver), so they are deliberately left out here.
- pytest -v -s models/kimi_k3
- label: ":nvidia: (H200) GLM5Next"
key: glm5next-unit-tests
+13 -1
View File
@@ -243,7 +243,19 @@ void cpu_attention_with_kv_cache(
input.block_table = block_table.data_ptr<int32_t>();
input.alibi_slopes =
alibi_slopes.has_value() ? alibi_slopes->data_ptr<float>() : nullptr;
input.s_aux = s_aux.has_value() ? s_aux->data_ptr<c10::BFloat16>() : nullptr;
// Attention sinks may be bf16 (native path) or fp32. Anything that is not
// bf16 must be provided as fp32 and is executed in full float precision.
if (s_aux.has_value()) {
TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 ||
s_aux->scalar_type() == at::ScalarType::Float,
"cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ",
s_aux->scalar_type(), " must be bfloat16 or float32");
input.s_aux = s_aux->data_ptr();
input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16;
} else {
input.s_aux = nullptr;
input.s_aux_is_bf16 = false;
}
input.dynamic_causal =
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
input.scale = scale;
+24 -9
View File
@@ -879,7 +879,11 @@ struct AttentionInput {
int32_t* seq_lens;
int32_t* block_table;
float* alibi_slopes;
c10::BFloat16* s_aux;
// Attention sinks pointer. May reference bf16 or fp32 data depending on
// `s_aux_is_bf16`. bf16 sinks keep the native bf16 path; anything else is
// provided as fp32 and executed in full float precision.
const void* s_aux;
bool s_aux_is_bf16;
bool* dynamic_causal;
float scale;
bool causal;
@@ -1565,7 +1569,8 @@ class AttentionMainLoop {
const float scale = input->scale;
const float softcap_scale = input->softcap;
const float* alibi_slopes = input->alibi_slopes;
const c10::BFloat16* s_aux = input->s_aux;
const void* s_aux = input->s_aux;
const bool s_aux_is_bf16 = input->s_aux_is_bf16;
const bool* dynamic_causal = input->dynamic_causal;
const bool is_dynamic_causal = dynamic_causal != nullptr;
@@ -1773,9 +1778,6 @@ class AttentionMainLoop {
const float* curr_alibi_slopes =
(alibi_slopes != nullptr ? alibi_slopes + q_head_start_idx
: nullptr);
const c10::BFloat16* curr_s_aux =
(s_aux != nullptr ? s_aux + q_head_start_idx : nullptr);
// copy the Q tile to q_buffer, the logical layout of q_buffer is
// [actual_q_token_num, curr_q_heads_per_kv, head_dim]
{
@@ -1787,10 +1789,23 @@ class AttentionMainLoop {
if (use_sink) {
alignas(64) float s_aux_fp32[16];
// All other platforms have BF16Vec16 available
vec_op::BF16Vec16 vec_bf16(curr_s_aux);
vec_op::FP32Vec16 vec_fp32(vec_bf16);
vec_fp32.save(s_aux_fp32);
// Sinks may be provided as bf16 or fp32. Either way they are
// loaded into fp32: bf16 sinks are up-converted (native
// BF16Vec16 path, available on all platforms), while fp32
// sinks are loaded directly and executed in full precision.
if (s_aux_is_bf16) {
const c10::BFloat16* curr_s_aux =
static_cast<const c10::BFloat16*>(s_aux) +
q_head_start_idx;
vec_op::BF16Vec16 vec_bf16(curr_s_aux);
vec_op::FP32Vec16 vec_fp32(vec_bf16);
vec_fp32.save(s_aux_fp32);
} else {
const float* curr_s_aux =
static_cast<const float*>(s_aux) + q_head_start_idx;
vec_op::FP32Vec16 vec_fp32(curr_s_aux);
vec_fp32.save(s_aux_fp32);
}
float* __restrict__ curr_sum_buffer = sum_buffer;
float* __restrict__ curr_max_buffer = max_buffer;
+3
View File
@@ -145,6 +145,9 @@ When batch invariance is enabled, vLLM:
1. Uses deterministic kernel implementations for attention and other operations
2. Ensures consistent numerical behavior across different batch sizes
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode)
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
!!! note
Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility.
+1
View File
@@ -549,6 +549,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
| `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | ✅︎ | ✅︎ |
| `DiffusionGemmaForBlockDiffusion` | DiffusionGemma | T + I<sup>+</sup> + V<sup>+</sup> | `google/diffusiongemma-26B-A4B-it`, etc. | | |
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
+1 -1
View File
@@ -33,4 +33,4 @@ quack-kernels==0.6.5 # Required by MSA
tokenspeed-mla==0.1.8; platform_system == "Linux"
# Humming kernels for quantization gemm
humming-kernels[cu13]==0.1.15
humming-kernels[cu13]==0.1.16
+4 -1
View File
@@ -128,8 +128,11 @@ FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
),
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
# The combined pass can consume these sites before the standalone pass.
"norm_rope_fusion": re.compile(
r"qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on (\d+) sites"
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
r"qk_norm_rope_kvcache_fusion.py:\d+] "
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
),
"attn_quant_fusion": re.compile(
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
+15 -2
View File
@@ -182,7 +182,11 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
# TODO: Remove log counting in unit tests
# once all matchers implement VllmFusionPatternMatcherPass
n_expected = tp_size * num_ranges_activated
if match_name not in ("attn_quant_fusion", "act_quant_fusion"):
if match_name not in (
"attn_quant_fusion",
"act_quant_fusion",
"norm_rope_fusion",
):
assert len(log_matches) == n_expected, (
f"Could not find {n_expected} {match_name} "
f"(found {len(log_matches)}) in:\n {log_holder.text}"
@@ -190,7 +194,16 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
expected_matches = getattr(matches, match_name)
if match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check:
if match_name == "norm_rope_fusion":
# Opaque LayerName lets the combined pass consume short-range
# sites before the standalone pass; count either owner.
assert len(log_matches) >= n_expected
assert all(m in (0, expected_matches) for m in log_matches)
assert sum(log_matches) == expected_matches * n_expected, (
f"{match_name} expected {expected_matches * n_expected} "
f"sites across both passes, found: {log_matches}"
)
elif match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check:
# AR+rms+quant takes precedence over rms+quant if activated.
# That means we get full matching where ar+rms+quant was not
# activated, and less where it was (only the smallest range).
@@ -8,6 +8,7 @@ import pytest
import torch
from packaging.version import Version
import vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion as fusion_module
import vllm.config
from tests.compile.backend import TestBackend
from tests.v1.attention.utils import (
@@ -15,8 +16,12 @@ from tests.v1.attention.utils import (
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
from vllm._aiter_ops import (
IS_AITER_FOUND,
is_aiter_found_and_supported,
rocm_aiter_ops,
)
from vllm.compilation.passes.fusion.matcher_utils import MROPE_OP, ROTARY_OP
from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import (
QkNormRopeKvCacheFusionPass,
)
@@ -37,8 +42,12 @@ from vllm.config import (
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
@@ -94,6 +103,7 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
device: torch.device,
rotary_dim: int | None = None,
prefix: str = "model.layers.0.self_attn.attn",
expected_query_quant_group_shape: GroupShape | None = None,
):
super().__init__()
self.num_heads = num_heads
@@ -133,6 +143,9 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
attn_backend=attn_backend.get_class(),
)
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
if expected_query_quant_group_shape is not None:
assert self.attn.query_quant is not None
assert self.attn.query_quant.group_shape == expected_query_quant_group_shape
assert not self.attn_backend.forward_includes_kv_cache_update, (
f"Attention backend {self.attn_backend} does not support "
"fuse_qk_norm_rope_kvcache."
@@ -227,18 +240,15 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
self.kv_cache_dtype != self.dtype
and self.attn.impl.supports_quant_query_input
):
q_fp8 = torch.empty_like(q, dtype=FP8_DTYPE)
torch.ops.vllm.rocm_aiter_per_tensor_quant(
q_fp8, q, self.attn._q_scale, False
)
q = q_fp8
assert self.attn.query_quant is not None
q, _ = self.attn.query_quant(q, self.attn._q_scale)
# Final views + KV cache update
q = q.view(-1, self.num_heads, self.head_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size)
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
k, v, self.layer_name
k, v, _encode_layer_name(self.layer_name)
)
return q, k, v, kv_cache_dummy_dep
@@ -260,6 +270,64 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default]
class QKNormMRoPEKVCacheTestModel(QKNormRoPEKVCacheTestModel):
def __init__(
self,
*,
mrope_section: tuple[int, int, int],
mrope_interleaved: bool,
**kwargs,
):
super().__init__(**kwargs)
self.rotary_emb = MRotaryEmbedding(
self.head_size,
rotary_dim=self.rotary_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=self.is_neox,
dtype=self.dtype,
mrope_section=list(mrope_section),
mrope_interleaved=mrope_interleaved,
)
self.enable_rope_custom_op = self.rotary_emb.enabled()
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v, kv_cache_dummy_dep = super().forward(qkv, positions)
output = torch.empty(
q.shape[0],
self.num_heads * self.head_size,
device=q.device,
dtype=self.dtype,
)
torch.ops.vllm.unified_attention_with_output(
q,
k,
v,
output,
_encode_layer_name(self.layer_name),
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
# Production decoder attention consumes K from the cache rather than
# its direct K argument. Returning `output` keeps that production user
# in the graph without exposing K to an unsupported consumer.
return q, output, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
return [MROPE_OP, torch.ops.vllm.unified_kv_cache_update.default]
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default]
class QKNormMRoPEObservableKTestModel(QKNormMRoPEKVCacheTestModel):
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return QKNormRoPEKVCacheTestModel.forward(self, qkv, positions)
def _run_qk_norm_rope_kvcache_fusion_test(
*,
attn_backend: AttentionBackendEnum,
@@ -278,18 +346,33 @@ def _run_qk_norm_rope_kvcache_fusion_test(
rms_norm_eps: float,
custom_op: str,
monkeypatch: pytest.MonkeyPatch,
mrope_section: tuple[int, int, int] | None = None,
mrope_interleaved: bool = False,
observe_mrope_k: bool = False,
expect_fusion: bool = True,
use_quark_scalar_query_scale: bool = False,
) -> None:
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
torch.set_default_device(device)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
# Set AITER toggles before VllmConfig initialization enables env caching.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
rocm_aiter_ops.refresh_env_variables()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
quant_config=(
QuarkConfig(quant_config={"exclude": []})
if use_quark_scalar_query_scale
else None
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=[custom_op],
@@ -302,6 +385,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_TRITON_ROPE",
"1" if enable_aiter_triton_rope else "0",
@@ -309,18 +393,49 @@ def _run_qk_norm_rope_kvcache_fusion_test(
m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout)
rocm_aiter_ops.refresh_env_variables()
model = QKNormRoPEKVCacheTestModel(
vllm_config=vllm_config,
attn_backend=attn_backend,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
rotary_dim=rotary_dim,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
dtype=dtype,
device=torch.get_default_device(),
)
expected_query_quant_group_shape = None
if use_quark_scalar_query_scale:
expected_query_quant_group_shape = (
GroupShape(-1, num_heads * head_size)
if num_kv_heads == 1
else GroupShape.PER_TENSOR
)
model_kwargs = {
"vllm_config": vllm_config,
"attn_backend": attn_backend,
"num_heads": num_heads,
"num_kv_heads": num_kv_heads,
"head_size": head_size,
"rotary_dim": rotary_dim,
"is_neox": is_neox,
"rms_norm_eps": rms_norm_eps,
"dtype": dtype,
"device": torch.get_default_device(),
"expected_query_quant_group_shape": expected_query_quant_group_shape,
}
if mrope_section is None:
model = QKNormRoPEKVCacheTestModel(**model_kwargs)
else:
alternate_section = (8, 8, 8)
m.setattr(
fusion_module,
"_discover_mrope_configs",
lambda _: (
(mrope_section, mrope_interleaved),
(alternate_section, not mrope_interleaved),
),
)
model_cls = (
QKNormMRoPEObservableKTestModel
if observe_mrope_k
else QKNormMRoPEKVCacheTestModel
)
model = model_cls(
**model_kwargs,
mrope_section=mrope_section,
mrope_interleaved=mrope_interleaved,
)
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
passes = [
@@ -337,7 +452,12 @@ def _run_qk_norm_rope_kvcache_fusion_test(
num_heads * head_size + 2 * num_kv_heads * head_size,
dtype=dtype,
)
pos = torch.arange(num_tokens, dtype=torch.long)
if mrope_section is None:
pos = torch.arange(num_tokens, dtype=torch.long)
else:
# Real Qwen-VL position IDs can be a strided [3, N] view.
pos = torch.randint(0, 4096, (3, 2 * num_tokens), dtype=torch.long)[:, ::2]
assert not pos.is_contiguous()
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
@@ -356,7 +476,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
# Run fused (compiled) forward
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
torch._dynamo.mark_dynamic(pos, 0 if mrope_section is None else 1)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
@@ -369,14 +489,33 @@ def _run_qk_norm_rope_kvcache_fusion_test(
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == 1
assert fusion_pass.matched_count == int(expect_fusion)
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
if expect_fusion:
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
else:
for op in model.ops_in_model_before():
assert backend.op_count(op, before=True) > 0
assert backend.op_count(op) > 0
if use_quark_scalar_query_scale:
expected_quant_op = (
torch.ops.vllm.rocm_aiter_per_tensor_quant.default
if num_kv_heads != 1
else torch.ops._C.static_scaled_fp8_quant.default
)
assert backend.op_count(expected_quant_op, before=True) > 0
# Query quantization remains separate after the prologue fusion.
assert backend.op_count(expected_quant_op) > 0
# Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2;
# AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2.
ATOL, RTOL = (2e-2, 2e-2) if enable_aiter_triton_rope else (1e-2, 1e-2)
ATOL, RTOL = (
(2e-2, 2e-2)
if enable_aiter_triton_rope or mrope_section is not None
else (1e-2, 1e-2)
)
is_fp8_cache = model.kv_cache_dtype != dtype
if q_fused.dtype == FP8_DTYPE:
@@ -387,7 +526,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
else:
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
if not is_fp8_cache:
if not expect_fusion or (not is_fp8_cache and mrope_section is None):
# The AITER PTS kernel populates k_out only for non-FP8 caches.
# With FP8, the kernel writes quantized K directly to the cache
# and may leave k_out uninitialised. In production this is fine
@@ -397,15 +536,16 @@ def _run_qk_norm_rope_kvcache_fusion_test(
# Should be bit exact since no processing had been done on v for both paths
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
# fp8 vs triton-rope ref requires loosening tolerance to 1.25e-1.
if is_fp8_cache and enable_aiter_triton_rope:
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
# Allow one E4M3 quantization step while still comparing every block.
if is_fp8_cache:
cache_atol = cache_rtol = 1.25e-1
else:
cache_atol, cache_rtol = ATOL, RTOL
torch.testing.assert_close(
kv_cache_unfused[0].float(),
kv_cache_fused[0].float(),
kv_cache_unfused.float(),
kv_cache_fused.float(),
atol=cache_atol,
rtol=cache_rtol,
)
@@ -494,3 +634,225 @@ def test_qk_norm_rope_kvcache_fusion(
custom_op=custom_op,
monkeypatch=monkeypatch,
)
@pytest.mark.parametrize(
"mrope_section",
[
pytest.param((16, 24, 24), id="full_rotary"),
pytest.param((4, 6, 6), id="partial_rotary"),
],
)
@pytest.mark.parametrize("mrope_interleaved", [False, True])
@pytest.mark.parametrize("is_neox", [False, True])
@pytest.mark.parametrize(
"kv_layout",
[
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
pytest.param(KVCacheLayout.LHBNC, id="block_major"),
],
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not IS_AITER_FOUND,
reason="Requires AITER",
)
def test_qk_norm_mrope_kvcache_fusion(
mrope_section: tuple[int, int, int],
mrope_interleaved: bool,
is_neox: bool,
kv_layout: KVCacheLayout,
kv_cache_dtype: str,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=2 * sum(mrope_section),
block_size=16,
is_neox=is_neox,
use_shuffle_kv_layout="0",
kv_layout=kv_layout,
dtype=torch.bfloat16,
kv_cache_dtype=kv_cache_dtype,
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=mrope_section,
mrope_interleaved=mrope_interleaved,
)
@pytest.mark.parametrize(
("num_heads", "num_kv_heads"),
[(32, 2), (16, 1), (8, 1)],
ids=["tp2", "tp4", "tp8"],
)
def test_qk_norm_mrope_kvcache_fusion_quark_scalar_query_scale(
num_heads: int,
num_kv_heads: int,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=128,
rotary_dim=128,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(24, 20, 20),
mrope_interleaved=True,
use_quark_scalar_query_scale=True,
)
def test_mrope_fusion_rejects_observable_k(monkeypatch: pytest.MonkeyPatch):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=128,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(16, 24, 24),
observe_mrope_k=True,
expect_fusion=False,
)
def test_mrope_fusion_rejects_unaligned_rotary_dim(
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=6,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(1, 1, 1),
expect_fusion=False,
)
@pytest.mark.skipif(not _USE_LAYERNAME, reason="requires opaque LayerName")
def test_opaque_layer_name_patterns_register_once(monkeypatch: pytest.MonkeyPatch):
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=torch.bfloat16),
cache_config=CacheConfig(block_size=16, cache_dtype="fp8_e4m3"),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rotary_embedding"],
pass_config=PassConfig(fuse_qk_norm_rope_kvcache=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
model_kwargs = {
"vllm_config": vllm_config,
"attn_backend": AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"num_heads": 8,
"num_kv_heads": 2,
"head_size": 128,
"is_neox": True,
"rms_norm_eps": 1e-6,
"dtype": torch.bfloat16,
"device": torch.device(device),
}
models = [
QKNormRoPEKVCacheTestModel(
**model_kwargs,
prefix=f"model.layers.{index}.self_attn.attn",
)
for index in range(2)
]
# Opaque layer names are wildcards in the pattern graph. Registering
# once per physical layer raises "Duplicate pattern" on torch >= 2.11.
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
assert models
assert fusion_pass.patterns
def _call_fused_mrope_impl(q_out: torch.Tensor) -> None:
device = q_out.device
fusion_module.fused_qk_norm_mrope_and_unified_kv_cache_update_impl(
q_out=q_out,
qkv=torch.zeros(2, 16, device=device),
positions=torch.arange(2, device=device),
q_weight=torch.ones(4, device=device),
k_weight=torch.ones(4, device=device),
rms_norm_eps=1e-6,
cos_sin_cache=torch.zeros(16, 4, device=device),
is_neox=True,
mrope_section_t=1,
mrope_section_h=1,
mrope_section_w=0,
is_interleaved=False,
rotary_dim=4,
layer_name="model.layers.0.self_attn.attn",
)
def test_mrope_profiling_pass_initializes_query(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
fusion_module,
"get_attention_context",
lambda _: (None, None, None, None),
)
q_out = torch.full((2, 8), torch.nan, device="cpu")
_call_fused_mrope_impl(q_out)
torch.testing.assert_close(q_out, torch.zeros_like(q_out))
def test_mrope_attention_context_failure_propagates(
monkeypatch: pytest.MonkeyPatch,
):
def fail_context(_):
raise RuntimeError("missing attention context")
monkeypatch.setattr(fusion_module, "get_attention_context", fail_context)
with pytest.raises(RuntimeError, match="missing attention context"):
_call_fused_mrope_impl(torch.empty(2, 8, device="cpu"))
@@ -14,8 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
@@ -24,6 +26,9 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
MODEL_NAME = "openai-community/gpt2"
MODEL_NAME_SHORT = "gpt2"
DEEPSEEK_V4_FIM_BEGIN = "<|fim▁begin|>"
DEEPSEEK_V4_FIM_HOLE = "<|fim▁hole|>"
DEEPSEEK_V4_FIM_END = "<|fim▁end|>"
_PER_REQUEST_STATS = RequestStateStats(
queued_ts=1.0,
scheduled_ts=1.5,
@@ -302,6 +307,179 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
assert response.metrics.speculative_decoding.num_spec_steps == 2
def _build_online_renderer_for_completion(
model_config: MockModelConfig,
renderer: Any | None = None,
) -> OnlineRenderer:
if renderer is None:
renderer = MagicMock()
renderer.render_completion_suffix.return_value = None
return OnlineRenderer(
model_config=model_config,
renderer=renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
def _deepseek_v4_suffix_renderer(
model_config: MockModelConfig,
) -> DeepseekV4Renderer:
return DeepseekV4Renderer(
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
None,
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_completion_suffix_uses_renderer_capability():
model_config = MockModelConfig()
online_renderer = _build_online_renderer_for_completion(
model_config,
renderer=_deepseek_v4_suffix_renderer(model_config),
)
online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
request = CompletionRequest(
model=MODEL_NAME,
prompt="def fib(n):\n return ",
suffix="\n\nprint(fib(10))",
max_tokens=64,
)
result = await online_renderer.render_completion(request)
assert result == [{"ok": True}]
online_renderer.preprocess_completion.assert_awaited_once()
assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == (
f"{DEEPSEEK_V4_FIM_BEGIN}def fib(n):\n return "
f"{DEEPSEEK_V4_FIM_HOLE}\n\nprint(fib(10))"
f"{DEEPSEEK_V4_FIM_END}"
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_completion_suffix_supports_text_prompt_list():
model_config = MockModelConfig()
online_renderer = _build_online_renderer_for_completion(
model_config,
renderer=_deepseek_v4_suffix_renderer(model_config),
)
online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
request = CompletionRequest(
model=MODEL_NAME,
prompt=["prefix A", "prefix B"],
suffix=" suffix",
)
result = await online_renderer.render_completion(request)
assert result == [{"ok": True}]
assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [
f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
]
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_completion_suffix_rejects_renderer_without_fim_support():
model_config = MockModelConfig()
online_renderer = _build_online_renderer_for_completion(model_config)
online_renderer.preprocess_completion = AsyncMock()
request = CompletionRequest(
model=MODEL_NAME,
prompt="prefix",
suffix="suffix",
)
result = await online_renderer.render_completion(request)
assert isinstance(result, ErrorResponse)
assert "FIM completion rendering" in result.error.message
online_renderer.preprocess_completion.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_completion_suffix_rejects_echo():
model_config = MockModelConfig()
online_renderer = _build_online_renderer_for_completion(
model_config,
renderer=_deepseek_v4_suffix_renderer(model_config),
)
online_renderer.preprocess_completion = AsyncMock()
request = CompletionRequest(
model=MODEL_NAME,
prompt="prefix",
suffix="suffix",
echo=True,
)
result = await online_renderer.render_completion(request)
assert isinstance(result, ErrorResponse)
assert "Echo is unsupported with suffix" in result.error.message
online_renderer.preprocess_completion.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_completion_suffix_rejects_prompt_embeds():
model_config = MockModelConfig()
online_renderer = _build_online_renderer_for_completion(
model_config,
renderer=_deepseek_v4_suffix_renderer(model_config),
)
online_renderer.preprocess_completion = AsyncMock()
request = CompletionRequest(
model=MODEL_NAME,
prompt="prefix",
suffix="suffix",
prompt_embeds=b"embeds",
)
result = await online_renderer.render_completion(request)
assert isinstance(result, ErrorResponse)
assert "suffix is not supported with prompt_embeds" in result.error.message
online_renderer.preprocess_completion.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("prompt", ([1, 2, 3], [[1, 2, 3]]))
async def test_completion_suffix_rejects_token_prompt(prompt):
model_config = MockModelConfig()
online_renderer = _build_online_renderer_for_completion(
model_config,
renderer=_deepseek_v4_suffix_renderer(model_config),
)
online_renderer.preprocess_completion = AsyncMock()
request = CompletionRequest(
model=MODEL_NAME,
prompt=prompt,
suffix="suffix",
)
result = await online_renderer.render_completion(request)
assert isinstance(result, ErrorResponse)
assert "requires text prompt input" in result.error.message
online_renderer.preprocess_completion.assert_not_awaited()
@pytest.mark.asyncio
async def test_completion_error_non_stream():
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
@@ -3,6 +3,5 @@ accuracy_threshold: 0.39
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
@@ -3,7 +3,6 @@ accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -1,5 +1,5 @@
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
accuracy_threshold: 0.78
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-a4"
accuracy_threshold: 0.70
num_questions: 1319
num_fewshot: 5
server_args: >-
@@ -0,0 +1,8 @@
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-asym"
accuracy_threshold: 0.85
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 4096
--linear-backend humming
@@ -1,3 +1,4 @@
gpt-oss-20b-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
@@ -4,4 +4,4 @@ Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
Qwen3-4B-mixed-quant-RTN-humming.yaml
Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
+49 -3
View File
@@ -428,6 +428,7 @@ def varlen_with_paged_kv(
k_scale: float = 1.0,
v_scale: float = 1.0,
dynamic_causal: list[bool] | None = None,
s_aux_dtype: torch.dtype = torch.bfloat16,
) -> None:
set_random_seed(0)
num_seqs = len(seq_lens)
@@ -449,9 +450,7 @@ def varlen_with_paged_kv(
# 2^(-8/n)
alibi_slopes = _get_alibi_slopes(num_query_heads) if use_alibi else None
s_aux = (
15 * torch.rand((num_query_heads,), dtype=torch.bfloat16) if use_sink else None
)
s_aux = 15 * torch.rand((num_query_heads,), dtype=s_aux_dtype) if use_sink else None
is_fp8 = kv_cache_dtype != "auto"
if is_fp8 and current_platform.get_cpu_architecture() != CpuArchEnum.X86:
@@ -538,6 +537,8 @@ def varlen_with_paged_kv(
)
out_without_split = torch.empty_like(query)
if s_aux is not None and s_aux.dtype != torch.bfloat16:
s_aux = s_aux.to(torch.float32)
cpu_attention_with_kv_cache(
query=query,
key_cache=packed_key_cache,
@@ -1128,6 +1129,51 @@ def test_varlen_with_paged_kv_sink(
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3"])
@pytest.mark.parametrize("seq_lens", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", [96])
@pytest.mark.parametrize("block_size", [128])
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("soft_cap", [None])
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
@pytest.mark.parametrize("use_alibi", [False])
@pytest.mark.parametrize("use_sink", [True])
@pytest.mark.parametrize("isa", [get_attn_isa()])
@pytest.mark.parametrize("s_aux_dtype", [torch.float16])
def test_varlen_with_paged_kv_sink_fp16(
seq_lens: list[tuple[int, int]],
num_heads: tuple[int, int],
head_size: int,
sliding_window: int | None,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
num_blocks: int,
use_alibi: bool,
use_sink: bool,
isa: str,
kv_cache_dtype: str,
s_aux_dtype: torch.dtype,
) -> None:
varlen_with_paged_kv(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=head_size,
sliding_window=sliding_window,
dtype=dtype,
block_size=block_size,
soft_cap=soft_cap,
num_blocks=num_blocks,
use_alibi=use_alibi,
use_sink=use_sink,
isa=isa,
kv_cache_dtype=kv_cache_dtype,
s_aux_dtype=s_aux_dtype,
)
@pytest.mark.parametrize(
"kv_cache_dtype",
[
+61 -1
View File
@@ -107,7 +107,16 @@ SEEDS = [0]
def torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False
a,
w1,
w2,
w1_s,
w2_s,
topk_weight,
topk_ids,
block_shape,
silu_fp32=False,
expert_map=None,
):
"""Fused MoE with block-wise fp8 quantization using native torch.
@@ -123,6 +132,9 @@ def torch_w8a8_block_fp8_moe(
topk_weight = topk_weight.view(-1)
topk_ids = topk_ids.view(-1)
if expert_map is not None:
# w1/w2 hold this rank's experts; slots mapped to -1 contribute nothing.
topk_ids = expert_map[topk_ids]
_, block_k = block_shape[0], block_shape[1]
# Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the
@@ -301,6 +313,54 @@ def test_w8a8_block_fp8_clamp_skips_fused_silu_quant(
)
@pytest.mark.parametrize(("M", "N", "K"), [(83, 512, 512), (2048, 4608, 512)])
@pytest.mark.parametrize("topk", [2, 6])
@pytest.mark.parametrize("ep_size", [2, 4])
@torch.inference_mode()
def test_w8a8_block_fp8_fused_moe_expert_map(M, N, K, topk, ep_size, workspace_init):
"""One EP rank: local expert shard plus an expert_map with -1 entries."""
E, block_size, dtype = 16, [128, 128], torch.bfloat16
torch.manual_seed(0)
a = torch.randn((M, K), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
w1, w2, quant_config = make_test_quant_config(
E, N, K, dtype, quant_dtype=torch.float8_e4m3fn, block_shape=block_size
)
local_e = E // ep_size
e_ids = torch.randperm(E, dtype=torch.int32)[:local_e]
e_map = torch.full((E,), -1, dtype=torch.int32)
e_map[e_ids] = torch.arange(local_e, dtype=torch.int32)
w1, w2 = w1[e_ids].contiguous(), w2[e_ids].contiguous()
w1_s = quant_config.w1_scale[e_ids].contiguous()
w2_s = quant_config.w2_scale[e_ids].contiguous()
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s, w2_scale=w2_s, block_shape=block_size
)
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size, expert_map=e_map
)
m_out = m_fused_moe.apply(
a,
w1,
w2,
topk_weights,
topk_ids,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=e_map,
global_num_experts=E,
)
torch.testing.assert_close(m_out, ref_out, atol=0.035, rtol=0.035)
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
@pytest.mark.parametrize("E", E)
@pytest.mark.parametrize("topk", TOP_KS)
@@ -339,6 +339,136 @@ def test_dsv4_fast_topk(
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
def test_dsv4_fast_topk_padding_uint32_falls_back(monkeypatch: pytest.MonkeyPatch):
"""Padded rows need the -1 sentinel, which uint32 cannot represent: the
router must skip the dsv4 fast path and still route the real rows."""
torch.manual_seed(0)
num_tokens = 17
num_experts = 256
hidden_states = torch.randn((num_tokens, 64), dtype=torch.float32, device="cuda")
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
is_padding = torch.zeros(num_tokens, dtype=torch.bool, device="cuda")
is_padding[1::2] = True
gating_output[is_padding] = float("nan")
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.router."
"fused_topk_bias_router._get_padding_mask",
lambda _: is_padding,
)
# uint32 + padding would trip dsv4_topk's signed-indices assertion; the
# generic path must take over instead.
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=correction_bias,
topk=6,
renormalize=True,
indices_type=torch.uint32,
routed_scaling_factor=1.5,
)
assert topk_ids.dtype == torch.uint32
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output[~is_padding],
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
)
# uint32 CUDA tensors do not support boolean-mask indexing; widen first.
torch.testing.assert_close(
topk_ids.to(torch.int64)[~is_padding],
topk_ids_ref.to(torch.int64),
atol=0,
rtol=0,
)
torch.testing.assert_close(
topk_weights[~is_padding], topk_weights_ref, atol=2e-5, rtol=2e-5
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
def test_dsv4_fast_topk_padding(monkeypatch: pytest.MonkeyPatch):
"""Verify the DSV4 fast path removes graph-padding rows from routing."""
torch.manual_seed(0)
num_tokens = 17
num_experts = 256
hidden_states = torch.randn((num_tokens, 64), dtype=torch.float32, device="cuda")
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
is_padding = torch.zeros(num_tokens, dtype=torch.bool, device="cuda")
is_padding[1::2] = True
gating_output[is_padding] = float("nan")
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.router."
"fused_topk_bias_router._get_padding_mask",
lambda _: is_padding,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=correction_bias,
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
)
assert torch.equal(topk_ids[is_padding], torch.full_like(topk_ids[is_padding], -1))
assert torch.equal(
topk_weights[is_padding], torch.zeros_like(topk_weights[is_padding])
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output[~is_padding],
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
)
torch.testing.assert_close(topk_ids[~is_padding], topk_ids_ref, atol=0, rtol=0)
torch.testing.assert_close(
topk_weights[~is_padding], topk_weights_ref, atol=2e-5, rtol=2e-5
)
# The mask buffer is persistent under CUDA graph replay, but its contents
# change with every batch. Verify that the kernel reads those contents at
# runtime rather than specializing on the mask captured above.
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_weights, graph_ids = dsv4_topk(
gating_output,
correction_bias,
torch.int32,
1.5,
is_padding=is_padding,
)
is_padding.logical_not_()
graph.replay()
assert torch.equal(
graph_ids[is_padding], torch.full_like(graph_ids[is_padding], -1)
)
assert torch.equal(
graph_weights[is_padding], torch.zeros_like(graph_weights[is_padding])
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
+23
View File
@@ -17,6 +17,26 @@ EPS = 1e-5
attn_res_module = importlib.import_module("vllm.models.kimi_k3.nvidia.ops.attn_res")
def _on_rocm_below_10() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import get_rocm_version
return (get_rocm_version() or (0,)) < (10,)
# The Triton bundled with ROCm < 10 (3.7.x) crashes in the AMD
# CanonicalizePointers pass on the kernel's tl.where over pointer tensors. Only
# the num_blocks > 0 loop contains it. ROCm serves AttnRes from
# vllm/models/kimi_k3/amd/ops/attn_res.py instead, so nothing real is lost.
_OLD_ROCM = _on_rocm_below_10()
def _skip_on_old_rocm(num_blocks: int) -> None:
if _OLD_ROCM and num_blocks > 0:
pytest.skip("Triton on ROCm < 10 cannot compile this kernel's pointer select")
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
storage = torch.randn(
*shape[:-1],
@@ -79,6 +99,7 @@ def test_attn_res(
):
if backend == "nvidia" and not current_platform.is_device_capability_family(100):
pytest.skip("NVIDIA AttnRes requires the SM100 family")
_skip_on_old_rocm(num_blocks)
prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
delta = (
@@ -133,6 +154,7 @@ def test_attn_res(
@pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1))
def test_attn_res_block_counts(num_blocks: int):
_skip_on_old_rocm(num_blocks)
prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
blocks = torch.randn(
1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
@@ -169,6 +191,7 @@ def test_attn_res_block_counts(num_blocks: int):
def test_attn_res_without_output_norm():
_skip_on_old_rocm(MAX_BLOCKS)
prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
delta = torch.randn_like(prefix)
blocks = torch.randn(
+16 -17
View File
@@ -855,7 +855,7 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
UnquantizedFusedMoEMethod,
)
dynamic_mxfp4_config = _make_qtensor_config(
mxfp4_config = _make_qtensor_config(
{
"dtype": "fp4",
"qscheme": "per_group",
@@ -866,24 +866,23 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
None,
exclude=["self_attn.q_proj", "mlp.down_proj"],
)
dynamic_mxfp4_config.dynamic_mxfp4_quant = True
assert dynamic_mxfp4_config.get_quant_method_target(
"self_attn.q_proj", LinearBase
) == (kMxfp4Static, None, QuarkLinearMethod)
attention_proj = TestLinear()
assert isinstance(
dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"),
QuarkLinearMethod,
assert mxfp4_config.get_quant_method_target("self_attn.q_proj", LinearBase) == (
None,
None,
UnquantizedLinearMethod,
)
assert isinstance(attention_proj.scheme, QuarkOCP_MX)
assert attention_proj.scheme.dynamic_mxfp4_quant
assert dynamic_mxfp4_config.get_quant_method_target(
"mlp.down_proj", LinearBase
) == (None, None, UnquantizedLinearMethod)
assert isinstance(
dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
mxfp4_config.get_quant_method(TestLinear(), "self_attn.q_proj"),
UnquantizedLinearMethod,
)
assert mxfp4_config.get_quant_method_target("mlp.down_proj", LinearBase) == (
None,
None,
UnquantizedLinearMethod,
)
assert isinstance(
mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
UnquantizedLinearMethod,
)
+66 -1
View File
@@ -12,6 +12,7 @@ from unittest.mock import patch
import pydantic
import pytest
import torch
from huggingface_hub import ResolvedRevision
from pydantic import ValidationError
@@ -37,7 +38,7 @@ from vllm.config import (
WatermarkConfig,
update_config,
)
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.config.compilation import CompilationMode, CUDAGraphMode, PassConfig
from vllm.config.kernel import IrOpPriorityConfig
from vllm.config.load import LoadConfig
from vllm.config.mamba import MambaBackendEnum
@@ -656,6 +657,70 @@ def test_breakable_cudagraph_platform_default(
default_breakable_cudagraph_architectures.cache_clear()
@pytest.mark.parametrize(
"case,hidden,heads,intermediate,tp,expected",
[
("bf16", 2048, (16, 8), 6144, 1, True),
("bi-off", 2048, (16, 8), 6144, 1, False),
("opt-out", 2048, (16, 8), 6144, 1, False),
("eager", 2048, (16, 8), 6144, 1, False),
("sp", 2048, (16, 8), 6144, 1, False),
("fp16", 2048, (16, 8), 6144, 1, False),
("quantized", 2048, (16, 8), 6144, 1, False),
("untuned", 4096, (32, 32), 11008, 1, False),
("tp4-tuned", 4096, (8, 2), 12288, 4, True),
("list-intermediate", 2048, (16, 8), [2048, 4096], 1, False),
("no-table", 2048, None, 6144, 1, False),
],
)
def test_batch_invariant_breakable_cudagraph(
monkeypatch, case, hidden, heads, intermediate, tp, expected
):
from vllm.config.vllm import default_breakable_cudagraph_architectures
from vllm.model_executor.determinism import batch_invariant_configs as bi_configs
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0" if case == "bi-off" else "1")
monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False)
if case == "opt-out":
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0")
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
table = None if case == "no-table" else {(12288, 2048): None, (6144, 4096): None}
monkeypatch.setattr(current_platform, "get_device_capability", lambda: None)
monkeypatch.setattr(bi_configs, "_get_tuned_matmul_arch_family", lambda cap: "test")
monkeypatch.setattr(
bi_configs,
"_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS",
{"test": table} if table else {},
)
default_breakable_cudagraph_architectures.cache_clear()
config = object.__new__(VllmConfig)
config.model_config = SimpleNamespace(
architectures=["Qwen3ForCausalLM"],
enforce_eager=case == "eager",
dtype=torch.float16 if case == "fp16" else torch.bfloat16,
quantization="fp8" if case == "quantized" else None,
hf_text_config=SimpleNamespace(intermediate_size=intermediate),
get_hidden_size=lambda: hidden,
get_head_size=lambda: 128,
get_num_attention_heads=lambda pc: heads[0],
get_num_kv_heads=lambda pc: heads[1],
)
config.parallel_config = SimpleNamespace(tensor_parallel_size=tp)
config.compilation_config = (
CompilationConfig(pass_config=PassConfig(enable_sp=True))
if case == "sp"
else CompilationConfig()
)
try:
assert config._maybe_enable_breakable_cudagraph() is expected
if expected:
assert config.compilation_config.mode == CompilationMode.NONE
finally:
os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None)
default_breakable_cudagraph_architectures.cache_clear()
@pytest.mark.parametrize(
("model_type", "expected_architecture"),
[
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake P/D accuracy with same-host HIP transport on ROCm."""
import pytest
from tests.utils import spawn_new_process_for_each_test
from tests.v1.kv_connector.rocm_pd_accuracy_utils import (
QWEN_MODEL,
run_rocm_pd_accuracy,
)
from vllm.platforms import current_platform
pytestmark = [
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"),
pytest.mark.distributed(num_gpus=4),
]
@pytest.mark.parametrize(
"model,prefill_tp,decode_tp",
[
pytest.param(QWEN_MODEL, 1, 1, id="mooncake-hip-qwen-p1-d1"),
pytest.param(QWEN_MODEL, 2, 2, id="mooncake-hip-qwen-p2-d2"),
],
)
@spawn_new_process_for_each_test
def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch):
run_rocm_pd_accuracy("mooncake", model, prefill_tp, decode_tp, monkeypatch)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL P/D accuracy with AITER unified attention on ROCm."""
import pytest
from tests.utils import spawn_new_process_for_each_test
from tests.v1.kv_connector.rocm_pd_accuracy_utils import (
DEEPSEEK_MODEL,
QWEN_MODEL,
run_rocm_pd_accuracy,
)
from vllm.platforms import current_platform
pytestmark = [
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"),
pytest.mark.distributed(num_gpus=4),
]
@pytest.mark.parametrize(
"model,prefill_tp,decode_tp",
[
pytest.param(QWEN_MODEL, 2, 2, id="nixl-aiter-qwen-p2-d2"),
pytest.param(QWEN_MODEL, 1, 2, id="nixl-aiter-qwen-p1-d2"),
pytest.param(QWEN_MODEL, 2, 1, id="nixl-aiter-qwen-p2-d1"),
pytest.param(DEEPSEEK_MODEL, 1, 1, id="nixl-aiter-deepseek-p1-d1"),
pytest.param(DEEPSEEK_MODEL, 1, 2, id="nixl-aiter-deepseek-p1-d2"),
pytest.param(DEEPSEEK_MODEL, 2, 1, id="nixl-aiter-deepseek-p2-d1"),
],
)
@spawn_new_process_for_each_test
def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch):
run_rocm_pd_accuracy("nixl", model, prefill_tp, decode_tp, monkeypatch)
@@ -0,0 +1,254 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared ROCm P/D accuracy harness for Mooncake and NIXL integration tests.
Keep the models, TP pairs, full GSM8K workload, and acceptance checks of the
Mooncake and FlashInfer/NIXL sweeps. Mooncake uses same-host HIP IPC; NIXL uses
AITER unified attention. These cases do not exercise RDMA or FlashInfer.
"""
import contextlib
import importlib
import json
import os
import signal
import subprocess
import sys
import time
from pathlib import Path
import requests
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
_ROOT = Path(__file__).resolve().parents[3]
QWEN_MODEL = "Qwen/Qwen3-0.6B"
DEEPSEEK_MODEL = "deepseek-ai/deepseek-vl2-tiny"
_REVISIONS = {
QWEN_MODEL: "c1899de289a04d12100db370d81485cdf75e47ca",
DEEPSEEK_MODEL: "66c54660eae7e90c9ba259bfdf92d07d6e3ce8aa",
}
def _metric_total(server, name: str, **labels: str) -> float:
response = requests.get(server.url_for("metrics"), timeout=10)
response.raise_for_status()
return sum(
sample.value
for family in text_string_to_metric_families(response.text)
for sample in family.samples
if sample.name == name
and all(sample.labels.get(key) == value for key, value in labels.items())
)
@contextlib.contextmanager
def _proxy(connector: str, prefill, decode, bootstrap_port: int):
port = get_open_port()
if connector == "mooncake":
script = (
_ROOT / "examples/disaggregated/mooncake_connector/"
"mooncake_connector_proxy.py"
)
args = [
"--prefill",
prefill.url_root,
str(bootstrap_port),
"--decode",
decode.url_root,
]
else:
script = Path(__file__).parent / "nixl_integration/toy_proxy_server.py"
args = [
"--prefiller-hosts",
"127.0.0.1",
"--prefiller-ports",
str(prefill.port),
"--decoder-hosts",
"127.0.0.1",
"--decoder-ports",
str(decode.port),
]
process = subprocess.Popen(
[sys.executable, str(script), "--port", str(port), *args],
start_new_session=True,
)
try:
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
assert process.poll() is None, "P/D proxy exited during startup"
try:
requests.get(f"http://127.0.0.1:{port}/", timeout=1)
break
except requests.ConnectionError:
time.sleep(0.2)
else:
raise TimeoutError("P/D proxy did not start")
yield f"http://127.0.0.1:{port}/v1"
finally:
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait(timeout=10)
def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
"""Transferred KV must feed decoding while full upstream accuracy holds."""
# Preserve the caller's allocation instead of querying host-wide SMI indices.
selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
selector = next((key for key in selectors if key in os.environ), selectors[0])
visible = os.getenv(selector)
devices = (
visible.split(",")
if visible is not None
else [str(i) for i in range(current_platform.device_count())]
)
assert len(devices) >= prefill_tp + decode_tp, "Insufficient allocated GPUs"
for key in selectors:
if key != selector:
monkeypatch.delenv(key, raising=False)
if connector == "mooncake":
import mooncake.engine
assert mooncake.engine.SUPPORT_HIP, "Mooncake requires its ROCm/HIP build"
connector_name = "MooncakeConnector"
extra = {"mooncake_protocol": "hip", "device_name": ""}
else:
from vllm.distributed.nixl_utils import NixlWrapper
assert NixlWrapper is not None, "NIXL ROCm bindings must import"
connector_name = "NixlConnector"
extra = {}
common_args = [
"--revision",
_REVISIONS[model],
"--dtype",
"bfloat16",
"--enforce-eager",
"--distributed-executor-backend",
"mp",
"--block-size",
"128",
"--max-num-seqs",
"128",
"--max-num-batched-tokens",
"4096",
"--gpu-memory-utilization",
"0.2",
"--kv-cache-memory-bytes",
str(8 * 1024**3),
"--moe-backend",
"triton",
"--attention-backend",
"ROCM_AITER_UNIFIED_ATTN",
]
common_env = {
"PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"],
"VLLM_HOST_IP": "127.0.0.1",
"VLLM_WORKER_MULTIPROC_METHOD": "spawn",
"VLLM_USE_V2_MODEL_RUNNER": "1",
"VLLM_ROCM_USE_AITER": "1",
"VLLM_KV_CACHE_LAYOUT": "HND",
}
bootstrap_port = get_open_port()
with contextlib.ExitStack() as stack:
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
for role, tp, assigned in (
("kv_producer", prefill_tp, devices[:prefill_tp]),
("kv_consumer", decode_tp, devices[prefill_tp : prefill_tp + decode_tp]),
):
config = {
"kv_connector": connector_name,
"kv_role": role,
"kv_connector_extra_config": extra,
}
env = {
**common_env,
selector: ",".join(assigned),
"VLLM_PORT": str(get_open_port()),
"VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()),
"VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port),
}
servers.append(
RemoteOpenAIServer(
model,
[
*common_args,
"--tensor-parallel-size",
str(tp),
"--kv-transfer-config",
json.dumps(config),
],
env_dict=env,
max_wait_seconds=1200,
)
)
prefill, decode = servers
proxy_url = stack.enter_context(
_proxy(connector, prefill, decode, bootstrap_port)
)
# Reuse the full upstream evaluation and its model-specific assertions.
accuracy = importlib.import_module(
f"tests.v1.kv_connector.{connector}_integration.test_accuracy"
)
monkeypatch.setattr(accuracy, "MODEL_NAME", model)
monkeypatch.setattr(accuracy, "BASE_URL", proxy_url)
assert model in accuracy.EXPECTED_VALUES
def transfer_totals():
totals = {
"external_tokens": _metric_total(
decode,
"vllm:prompt_tokens_by_source_total",
source="external_kv_transfer",
)
}
if connector == "nixl":
totals["nixl_bytes"] = _metric_total(
decode, "vllm:nixl_bytes_transferred_sum"
)
return totals
simple_evaluate = accuracy.lm_eval.simple_evaluate
def evaluate_with_transfer_checks(*args, **kwargs):
# The upstream helper runs a smoke prompt before GSM8K. Wait until
# its counters appear, then require new transfers during evaluation.
deadline = time.monotonic() + 15
before = transfer_totals()
while not all(value > 0 for value in before.values()):
assert time.monotonic() < deadline, (
f"Smoke prompt did not transfer KV: {before}"
)
time.sleep(1)
before = transfer_totals()
# The API adapter loads a local tokenizer because these proxies do
# not expose /tokenize. Match it to the server's pinned checkpoint.
kwargs["model_args"] += f",revision={_REVISIONS[model]}"
results = simple_evaluate(*args, **kwargs)
deadline = time.monotonic() + 15
after = transfer_totals()
while not all(after[key] > value for key, value in before.items()):
assert time.monotonic() < deadline, (
f"GSM8K did not transfer KV: before={before}, after={after}"
)
time.sleep(1)
after = transfer_totals()
print(f"GSM8K transfer counters: before={before}, after={after}")
return results
monkeypatch.setattr(
accuracy.lm_eval, "simple_evaluate", evaluate_with_transfer_checks
)
accuracy.test_accuracy()
@@ -50,7 +50,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
NixlKVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
ReadSpec,
TPMapping,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
get_base_request_id,
)
@@ -344,6 +347,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
)
w._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list)
w._send_failures = set()
w._sending_transfers_lock = threading.Lock()
w._push_finished_blocks = {}
w._pending_d_registrations = {}
@@ -941,9 +945,10 @@ class TestPushWriterNotifs:
assert request_id not in w._sending_transfers
@pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)])
def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma):
"""P-side sends are completed by consumer notifs / lease expiry,
not by local WRITE-handle completion."""
def test_completed_push_send_releases_blocks_without_consumer_notif(
self, pp_size, is_hma
):
"""All WRITEs completing lets P reclaim its blocks without a D ACK."""
w = self._pollable_worker()
w.pp_size = pp_size
w._is_hma_required = is_hma
@@ -951,20 +956,14 @@ class TestPushWriterNotifs:
w.nixl_wrapper.check_xfer_state.side_effect = ["DONE", "PROC", "DONE"]
results = w.get_transfer_results()
assert request_id not in results.finished_sending
assert request_id in w._reqs_to_send
assert w._sending_transfers[request_id] == [102]
assert request_id in w._reqs_to_send
assert w._evict_finished_inbox.empty()
results = w.get_transfer_results()
assert request_id not in results.finished_sending
assert request_id in w._reqs_to_send
assert request_id not in w._sending_transfers
# The consumer notif is what completes the send.
w._pending_completion_notifs.put(f"{request_id}:1".encode())
results = w.get_transfer_results()
assert request_id in results.finished_sending
assert results.finished_sending == {request_id}
assert request_id not in w._reqs_to_send
assert request_id not in w._reqs_to_process
assert w._evict_finished_inbox.get_nowait() == request_id
@@ -973,6 +972,91 @@ class TestPushWriterNotifs:
assert w.nixl_wrapper.release_xfer_handle.call_count == 2
w.xfer_stats.record_kv_expired_req.assert_not_called()
@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"])
def test_push_failure_survives_until_lease_expiry(self, sibling_state):
"""A later sibling completion must not turn a failed send into success."""
w = self._pollable_worker()
request_id = self._make_sending_req(w)
w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state]
assert w.get_transfer_results().finished_sending == set()
assert w._sending_transfers[request_id] == [102]
assert w.get_transfer_results().finished_sending == set()
assert request_id not in w._sending_transfers
assert request_id in w._reqs_to_send
assert w._evict_finished_inbox.empty()
w.expected_consumer_notifications_by_req = {}
w._reqs_to_send[request_id] = time.perf_counter() - 1
assert w.get_transfer_results().finished_sending == {request_id}
assert request_id not in w._send_failures
assert w._evict_finished_inbox.get_nowait() == request_id
@pytest.mark.parametrize("release_fails", [False, True])
def test_push_submission_failure_is_remembered_after_sibling_success(
self, release_fails
):
"""A submission error remains a failure even if later polls succeed."""
w = self._pollable_worker()
request_id = self._make_sending_req(w)
w.transfer_topo.block_size_ratio.return_value = 1
w._apply_prefix_caching = MagicMock(return_value=([[1]], [[1]]))
w._compute_desc_ids = MagicMock(return_value=[0])
w.dst_num_blocks = {w.engine_id: 8}
w.dst_region_num_blocks[w.engine_id] = [8]
w.dst_region_group_ids[w.engine_id] = [0]
w.dst_uses_region_group_mapping[w.engine_id] = False
w.nixl_wrapper.make_prepped_xfer.return_value = 101
w.nixl_wrapper.transfer.side_effect = RuntimeError("submit")
if release_fails:
w.nixl_wrapper.release_xfer_handle.side_effect = [
RuntimeError("release"),
None,
None,
]
handle = w._xfer_blocks(
read_spec=ReadSpec(0, [[1]], [[1]]),
dst_engine_id=w.engine_id,
request_id=request_id,
remote_request_id="decode-request",
local_xfer_side_handle=1,
remote_xfer_side_handle=2,
)
assert handle == (101 if release_fails else None)
w._sending_transfers[request_id] = [102]
if handle is not None:
w._sending_transfers[request_id].append(handle)
w.nixl_wrapper.check_xfer_state.return_value = "DONE"
assert w.get_transfer_results().finished_sending == set()
assert request_id in w._reqs_to_send
assert request_id not in w._sending_transfers
assert w._evict_finished_inbox.empty()
@pytest.mark.parametrize("first_state", ["ERR", "DONE"])
def test_push_completion_after_expiry_is_not_reported_twice(self, first_state):
"""A WRITE finishing after lease expiry must not complete the request again."""
w = self._pollable_worker()
request_id = self._make_sending_req(w)
w.nixl_wrapper.check_xfer_state.side_effect = [
first_state,
"PROC",
"PROC",
"DONE",
]
assert w.get_transfer_results().finished_sending == set()
w.expected_consumer_notifications_by_req = {}
w._reqs_to_send[request_id] = time.perf_counter() - 1
assert w.get_transfer_results().finished_sending == {request_id}
assert w._evict_finished_inbox.get_nowait() == request_id
assert w.get_transfer_results().finished_sending == set()
assert request_id not in w._sending_transfers
assert request_id not in w._send_failures
assert w._evict_finished_inbox.empty()
# ----------------------------------------------------------------- #
# Negative / error-path tests #
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from types import SimpleNamespace
import numpy as np
@@ -8,7 +9,9 @@ import pytest
import torch
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.spec_decode.dflash import speculator as dflash_speculator
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
DFlashSpeculator,
prepare_dflash_inputs,
)
@@ -33,6 +36,7 @@ def _run_prepare(
input_buffers = SimpleNamespace(
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
query_start_loc=torch.full(
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
),
@@ -140,6 +144,114 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
assert out.temperature[2].item() == 1.0
assert out.seeds[2].item() == 17
assert not out.input_buffers.is_padding[:3].any()
assert out.input_buffers.is_padding[3:].all()
assert out.input_buffers.input_ids[3:].cpu().tolist() == [0] * 13
assert out.input_buffers.positions[3:].cpu().tolist() == [0] * 13
def test_prepare_dflash_inputs_compacts_noncontiguous_request_slots():
device = torch.device("cuda")
max_num_reqs = 4
max_num_tokens = 16
num_speculative_steps = 3
input_buffers = SimpleNamespace(
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
query_start_loc=torch.full(
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
),
seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device),
)
input_batch = SimpleNamespace(
num_reqs=2,
num_scheduled_tokens=np.array([4, 4], dtype=np.int32),
positions=torch.tensor(
[10, 11, 12, 13, 20, 21, 22, 23],
dtype=torch.int64,
device=device,
),
query_start_loc=torch.tensor([0, 4, 8], dtype=torch.int32, device=device),
# Active batch rows are compact, while request state remains in slots 3 and 1.
idx_mapping=torch.tensor([3, 1], dtype=torch.int32, device=device),
)
query_slot_mapping = torch.full(
(max_num_tokens,), -2, dtype=torch.int64, device=device
)
context_positions = torch.full(
(max_num_tokens,), -1, dtype=torch.int64, device=device
)
context_slot_mapping = torch.full(
(max_num_tokens,), -2, dtype=torch.int64, device=device
)
sample_indices = torch.full(
(max_num_reqs * num_speculative_steps,),
-1,
dtype=torch.int64,
device=device,
)
sample_pos = torch.full_like(sample_indices, -1)
sample_idx_mapping = torch.full(
sample_indices.shape, -1, dtype=torch.int32, device=device
)
temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device)
seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device)
input_temperature = torch.tensor(
[0.0, 0.5, 0.0, 1.0], dtype=torch.float32, device=device
)
input_seeds = torch.tensor([0, 11, 0, 33], dtype=torch.int64, device=device)
last_sampled = torch.tensor([0, 77, 0, 99], dtype=torch.int64, device=device)
next_prefill_tokens = torch.zeros_like(last_sampled)
block_table = torch.tensor(
[[0, 0, 7, 8, 9, 10, 11, 12], [0, 0, 13, 14, 15, 16, 17, 18]],
dtype=torch.int32,
device=device,
)
prepare_dflash_inputs(
input_buffers,
query_slot_mapping,
context_positions,
context_slot_mapping,
sample_indices,
sample_pos,
sample_idx_mapping,
temperature,
seeds,
input_batch,
torch.tensor([1, 1], dtype=torch.int32, device=device),
torch.tensor([2, 1], dtype=torch.int32, device=device),
last_sampled,
next_prefill_tokens,
input_temperature,
input_seeds,
block_table,
4,
0,
1,
1,
123,
num_speculative_steps,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
128,
sample_from_anchor=True,
)
torch.accelerator.synchronize()
# Query rows follow compact batch order, but every persistent state lookup
# follows idx_mapping instead of accidentally using the compact row index.
assert input_buffers.input_ids[:6].cpu().tolist() == [99, 123, 123, 77, 123, 123]
assert input_buffers.positions[:6].cpu().tolist() == [12, 13, 14, 23, 24, 25]
assert sample_indices[:6].cpu().tolist() == [0, 1, 2, 3, 4, 5]
assert sample_idx_mapping[:6].cpu().tolist() == [3, 3, 3, 1, 1, 1]
assert temperature.cpu().tolist() == [0.0, 0.5, 0.0, 1.0]
assert seeds.cpu().tolist() == [0, 11, 0, 33]
assert not input_buffers.is_padding[:6].any()
assert input_buffers.is_padding[6:].all()
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
out = _run_prepare(
@@ -174,3 +286,40 @@ def test_prepare_dflash_inputs_never_writes_the_null_block():
PAD_SLOT_ID,
PAD_SLOT_ID,
]
def test_dflash_forward_context_receives_draft_padding_mask(monkeypatch):
device = torch.device("cuda")
input_buffers = SimpleNamespace(
input_ids=torch.tensor([11, 12, 0, 0], dtype=torch.int32, device=device),
positions=torch.tensor([7, 8, 0, 0], dtype=torch.int64, device=device),
is_padding=torch.tensor([False, False, True, True], device=device),
)
observed = None
@contextmanager
def fake_set_forward_context(*args, **kwargs):
nonlocal observed
observed = kwargs["is_padding"].clone()
yield
monkeypatch.setattr(
dflash_speculator, "set_forward_context", fake_set_forward_context
)
speculator = SimpleNamespace(
input_buffers=input_buffers,
vllm_config=SimpleNamespace(),
model=lambda **kwargs: kwargs["input_ids"],
)
result = DFlashSpeculator._run_model(
speculator,
num_tokens=4,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=None,
)
assert result.tolist() == [11, 12, 0, 0]
assert observed is not None
assert observed.tolist() == [False, False, True, True]
+57
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.platforms import current_platform
from vllm.platforms.interface import Platform
from vllm.sampling_params import SamplingParams
from vllm.utils.mem_constants import GiB_bytes
from vllm.utils.system_utils import update_environment_variables
@@ -425,6 +426,62 @@ def test_select_common_block_size_no_valid_option():
select_common_block_size(48, [backend_a, backend_b])
def _mock_backend(supported: list, *, exact: bool = False):
"""Backend accepting multiples of ``supported``, or only those exact sizes
when ``exact`` (as CPU_MLA does)."""
from vllm.v1.attention.backend import AttentionBackend
class _MockBackendCls(AttentionBackend):
@staticmethod
def get_name() -> str:
return "MOCK_EXACT" if exact else "MOCK"
@staticmethod
def get_supported_kernel_block_sizes():
return list(supported)
if exact:
@classmethod
def supports_block_size(cls, block_size: int | None) -> bool:
return block_size is None or block_size in supported
return _MockBackendCls
@pytest.mark.parametrize(
"backends,expected",
[
# A lone backend keeps the default; MultipleOf(8) makes a
# minimum-based regression return 8 instead of 16.
([[MultipleOf(8)]], 16),
# Sparse-MLA main backend (32 or 64) beside an indexer supporting only
# 64: the main backend alone picks 32, which the indexer rejects at
# select_common_block_size time.
([[32, 64], [64]], 64),
([[32, 64]], 32),
# Neither divides the other, so only the LCM satisfies both.
([[32], [48]], 96),
],
)
def test_preferred_block_size_satisfies_every_backend(backends, expected):
classes = [_mock_backend(s) for s in backends]
assert Platform._preferred_block_size_for_backends(classes, 16, None) == expected
def test_preferred_block_size_searches_past_an_exact_size_backend():
# Extending greedily picks lcm(16, 32) = 32, which the exact backend
# rejects; 96 is accepted by both.
classes = [_mock_backend([16, 96], exact=True), _mock_backend([MultipleOf(32)])]
assert Platform._preferred_block_size_for_backends(classes, 16, None) == 96
def test_preferred_block_size_rejects_backends_with_no_common_size():
classes = [_mock_backend([16], exact=True), _mock_backend([MultipleOf(64)])]
with pytest.raises(ValueError, match="share no supported KV cache block size"):
Platform._preferred_block_size_for_backends(classes, 16, None)
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
model = Mock()
model.get_mm_lora_token_counts.side_effect = (
+149 -3
View File
@@ -5,16 +5,35 @@
from unittest.mock import Mock
import numpy as np
import pytest
import torch
from vllm.v1.worker.gpu import pp_utils
from vllm.v1.worker.gpu import model_runner, pp_utils
def _batch(num_computed, prefill_len, num_scheduled):
def _cuda_handler(max_sample_len=6):
handler = object.__new__(pp_utils.PPHandler)
handler.is_last_rank = True
handler.disabled = False
handler.max_sample_len = max_sample_len
handler.last_rank = 1
handler.broadcast_group = Mock()
handler.device = torch.device("cuda")
handler.main_stream = torch.cuda.current_stream()
handler.broadcast_stream = torch.cuda.Stream()
return handler
def _batch(num_computed, prefill_len, num_scheduled, idx_mapping=None):
num_reqs = len(num_computed)
if idx_mapping is None:
idx_mapping = list(range(num_reqs))
return Mock(
num_reqs=len(num_computed),
num_reqs=num_reqs,
num_computed_tokens_np=np.array(num_computed, dtype=np.int32),
prefill_len_np=np.array(prefill_len, dtype=np.int32),
num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32),
idx_mapping=torch.tensor(idx_mapping, dtype=torch.int64),
)
@@ -81,3 +100,130 @@ def test_decode_row_ahead_of_a_prefill_chunk():
assert mask is not None
assert mask.tolist() == [True, False]
def test_disabled_handler_skips_broadcast_and_receive(monkeypatch):
"""While disabled (warmup), neither side enqueues a broadcast op."""
sent = []
monkeypatch.setattr(
pp_utils.torch.distributed,
"broadcast",
lambda *args, **kwargs: sent.append((args, kwargs)),
)
handler = object.__new__(pp_utils.PPHandler)
handler.set_disabled(True)
handler.is_last_rank = False
assert handler.receive(Mock()) is False
handler.is_last_rank = True
assert handler.broadcast(Mock(), Mock(), Mock(), Mock()) is None
assert sent == []
handler.set_disabled(False)
assert handler.disabled is False
def test_alloc_combined_keeps_unbind_views_16_byte_aligned():
"""Triton specializes on pointer alignment: an unaligned `num_rejected`
would compile a second `_post_update_kernel` variant at serving time,
where the in-flight broadcast NCCL kernel can block the module load."""
for num_reqs in range(1, 9):
combined = pp_utils._alloc_combined(num_reqs, torch.device("cpu"))
num_sampled, num_rejected = combined.unbind(dim=0)
assert num_sampled.data_ptr() % 16 == 0
assert num_rejected.data_ptr() % 16 == 0
assert combined.shape[1] >= num_reqs
def test_warmup_pp_decode_update_matches_serving_specialization(monkeypatch):
"""The warmup launch must hit the same triton specialization as serving.
A mismatch means the first real ``update_pp_decode_requests`` recompiles
mid-serving, where the in-flight broadcast NCCL kernel blocks the CUDA
module load and deadlocks the pipeline.
"""
calls = []
monkeypatch.setattr(model_runner, "post_update", lambda *args: calls.append(args))
runner = object.__new__(model_runner.GPUModelRunner)
runner.device = torch.device("cpu")
runner.pp_handler = Mock(max_sample_len=3)
runner.req_states = Mock()
runner.warmup_pp_decode_update()
assert len(calls) == 1
args = calls[0]
idx_mapping, _, _, output_bin_counts = args[:4]
sampled_tokens, num_sampled, num_rejected, query_start_loc = args[4:8]
broadcast_drafts, draft_tokens_out = args[10:12]
assert idx_mapping.tolist() == [-1] and idx_mapping.dtype == torch.int64
assert output_bin_counts is None
assert query_start_loc is None
assert sampled_tokens.shape == (1, 3) and sampled_tokens.dtype == torch.int64
assert num_sampled.dtype == torch.int32
assert num_rejected.dtype == torch.int32
# Spec-enabled PP handlers receive drafts over the broadcast; the warmup
# must compile that specialization (non-None draft pointers) too.
assert broadcast_drafts.shape == (1, 2) and broadcast_drafts.dtype == torch.int64
assert draft_tokens_out is runner.req_states.draft_tokens
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA stream")
def test_broadcast_pads_plain_sampler_rows_to_max_sample_len(monkeypatch):
"""The wire shape must not depend on whether the batch carried drafts:
the receiver always allocates [num_reqs, max_sample_len], and a NCCL
broadcast with mismatched counts hangs the receiver."""
sent = []
monkeypatch.setattr(
pp_utils.torch.distributed,
"broadcast",
lambda tensor, **kw: sent.append(tensor),
)
handler = _cuda_handler()
batch = _batch(num_computed=[10], prefill_len=[8], num_scheduled=[1])
handler.broadcast(
torch.zeros(1, 1, dtype=torch.int64, device="cuda"), # plain sampler
torch.ones(1, dtype=torch.int32, device="cuda"),
torch.zeros(1, dtype=torch.int32, device="cuda"),
batch,
)
assert sent[0].shape == (1, 6)
assert sent[1].shape == (2, 4)
torch.accelerator.synchronize()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA stream")
def test_broadcast_drafts_gathers_fresh_rows_from_the_table(monkeypatch):
sent = []
monkeypatch.setattr(
pp_utils.torch.distributed,
"broadcast",
lambda tensor, **kw: sent.append(tensor),
)
handler = _cuda_handler()
# The batch's single row maps to request-state row 2.
batch = _batch(num_computed=[10], prefill_len=[8], num_scheduled=[1])
batch.idx_mapping = torch.tensor([2], dtype=torch.int64, device="cuda")
table = torch.arange(20, dtype=torch.int64, device="cuda").view(4, 5)
handler.broadcast_drafts(table, batch)
assert sent[0].shape == (1, 5)
# The payload is a gather into a fresh tensor: propose() overwrites its
# persistent buffer on the next step, possibly before this send completes.
assert sent[0].data_ptr() != table.data_ptr()
assert sent[0].cpu().tolist() == [table[2].cpu().tolist()]
# An all-prefill batch sends nothing (receive() enqueues nothing either).
sent.clear()
prefill_batch = _batch(num_computed=[0], prefill_len=[4096], num_scheduled=[448])
prefill_batch.idx_mapping = prefill_batch.idx_mapping.cuda()
handler.broadcast_drafts(table, prefill_batch)
assert sent == []
torch.accelerator.synchronize()
+109
View File
@@ -3362,6 +3362,115 @@ class rocm_aiter_ops:
rotary_dim=kernel_rotary_dim,
)
@staticmethod
def fused_qk_norm_mrope_and_cache(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
num_heads_q: int,
num_heads_k: int,
head_dim: int,
is_neox: bool,
mrope_section: list[int],
is_interleaved: bool,
rms_norm_eps: float,
q_out: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
slot_mapping: torch.Tensor,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
block_size: int,
x: int,
rotary_dim: int = 0,
) -> None:
from aiter.ops.fused_qk_norm_mrope_cache_quant import (
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle,
)
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
qkv,
q_weight,
k_weight,
cos_sin_cache,
positions,
qkv.size(0),
num_heads_q,
num_heads_k,
num_heads_k,
head_dim,
is_neox,
mrope_section,
is_interleaved,
rms_norm_eps,
q_out,
k_cache,
v_cache,
slot_mapping,
k_scale,
v_scale,
None,
None,
False,
False,
block_size,
x,
rotary_dim,
)
@staticmethod
def do_qk_norm_mrope_kvcache_update(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
num_heads_q: int,
num_heads_k: int,
head_dim: int,
is_neox: bool,
mrope_section: list[int],
is_interleaved: bool,
rms_norm_eps: float,
q_out: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
slot_mapping: torch.Tensor,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
kv_cache_dtype: str,
rotary_dim: int = 0,
) -> None:
if kv_cache_dtype.startswith("fp8"):
key_cache = key_cache.view(current_platform.fp8_dtype())
value_cache = value_cache.view(current_platform.fp8_dtype())
rocm_aiter_ops.fused_qk_norm_mrope_and_cache(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=num_heads_q,
num_heads_k=num_heads_k,
head_dim=head_dim,
is_neox=is_neox,
mrope_section=mrope_section,
is_interleaved=is_interleaved,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_cache=key_cache,
v_cache=value_cache,
slot_mapping=slot_mapping,
k_scale=k_scale,
v_scale=v_scale,
block_size=key_cache.shape[1],
x=16 // key_cache.element_size(),
rotary_dim=rotary_dim,
)
@staticmethod
def triton_rope_and_cache(
query: torch.Tensor,
@@ -31,6 +31,7 @@ from vllm.platforms import current_platform
ROTARY_OP = torch.ops._C.rotary_embedding.default
FLASHINFER_ROTARY_OP = torch.ops.vllm.flashinfer_rotary_embedding.default
MROPE_OP = torch.ops.vllm.mrope.default
QUANT_OPS: dict[QuantKey, OpOverload] = {
kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa: E501
@@ -162,6 +163,74 @@ class MatcherRotaryEmbedding(MatcherCustomOp):
return result
class MatcherMRotaryEmbedding(MatcherCustomOp):
def __init__(
self,
is_neox: bool,
head_size: int,
rotary_dim: int,
num_heads: int,
num_kv_heads: int,
mrope_section: tuple[int, int, int],
mrope_interleaved: bool,
enabled: bool | None = None,
) -> None:
if enabled is None:
enabled = RotaryEmbedding.enabled()
super().__init__(enabled)
self.is_neox = is_neox
self.head_size = head_size
self.rotary_dim = rotary_dim
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.q_size = self.num_heads * self.head_size
self.kv_size = self.num_kv_heads * self.head_size
self.mrope_section = mrope_section
self.mrope_interleaved = mrope_interleaved
def inputs(self) -> list[torch.Tensor]:
positions = self.empty_int64(3, 5)
query = self.empty(5, self.q_size)
key = self.empty(5, self.kv_size)
cos_sin_cache = self.empty(4096, self.rotary_dim)
return [positions, query, key, cos_sin_cache]
def forward_custom(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
cos_sin_cache: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
result = auto_functionalized(
MROPE_OP,
positions=positions,
query=query,
key=key,
cos_sin_cache=cos_sin_cache,
head_size=self.head_size,
rotary_dim=self.rotary_dim,
mrope_section_t=self.mrope_section[0],
mrope_section_h=self.mrope_section[1],
mrope_section_w=self.mrope_section[2],
mrope_interleaved=self.mrope_interleaved,
is_neox_style=self.is_neox,
)
return result[1], result[2]
def forward_native(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
cos_sin_cache: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
raise NotImplementedError(
"MRoPE fusion requires the rotary_embedding custom op"
)
class MatcherRMSNormGated(MatcherCustomOp):
"""Matches RMSNormGated with norm_before_gate=True and group_size=None."""
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Callable
from collections.abc import Callable, Mapping
from typing import ParamSpec
import torch
@@ -19,12 +19,19 @@ from vllm.model_executor.layers.attention.attention import (
Attention,
get_attention_context,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.utils.torch_utils import (
_USE_LAYERNAME,
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
from .matcher_utils import MatcherRotaryEmbedding
from .matcher_utils import MROPE_OP, MatcherMRotaryEmbedding, MatcherRotaryEmbedding
from .rms_quant_fusion import empty_bf16, empty_fp32, empty_i64
logger = init_logger(__name__)
@@ -51,8 +58,9 @@ def fused_qk_norm_rope_and_unified_kv_cache_update_impl(
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
layer_name: str = "",
layer_name: LayerNameType,
) -> torch.Tensor:
layer_name = _resolve_layer_name(layer_name)
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
if layer_slot_mapping is not None:
attn_layer.impl.do_qk_norm_rope_kvcache_update(
@@ -87,7 +95,7 @@ def fused_qk_norm_rope_and_unified_kv_cache_update_fake(
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
layer_name: str = "",
layer_name: LayerNameType,
) -> torch.Tensor:
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
@@ -100,6 +108,76 @@ direct_register_custom_op(
)
def fused_qk_norm_mrope_and_unified_kv_cache_update_impl(
q_out: torch.Tensor,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
mrope_section_t: int,
mrope_section_h: int,
mrope_section_w: int,
is_interleaved: bool,
rotary_dim: int,
layer_name: LayerNameType,
) -> torch.Tensor:
layer_name = _resolve_layer_name(layer_name)
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
if layer_slot_mapping is not None:
if positions.ndim == 1:
positions = positions.unsqueeze(0).expand(3, -1).contiguous()
attn_layer.impl.do_qk_norm_mrope_kvcache_update(
attn_layer,
qkv,
q_out,
positions,
q_weight,
k_weight,
rms_norm_eps,
cos_sin_cache,
is_neox,
(mrope_section_t, mrope_section_h, mrope_section_w),
is_interleaved,
rotary_dim,
kv_cache,
layer_slot_mapping,
)
else:
q_out.zero_()
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
def fused_qk_norm_mrope_and_unified_kv_cache_update_fake(
q_out: torch.Tensor,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
mrope_section_t: int,
mrope_section_h: int,
mrope_section_w: int,
is_interleaved: bool,
rotary_dim: int,
layer_name: LayerNameType,
) -> torch.Tensor:
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
direct_register_custom_op(
op_name="fused_qk_norm_mrope_and_unified_kv_cache_update",
op_func=fused_qk_norm_mrope_and_unified_kv_cache_update_impl,
mutates_args=["q_out"],
fake_impl=fused_qk_norm_mrope_and_unified_kv_cache_update_fake,
)
# ---------------------------------------------------------------------------
# Pattern: QK-norm + RoPE + unified_kv_cache_update
# ---------------------------------------------------------------------------
@@ -142,19 +220,27 @@ class QkNormRopeKvCachePattern:
self.eps = eps
self.is_neox = is_neox
self.quant_query = quant_query
self.encoded_layer_name = _encode_layer_name(self.layer_name)
self.query_quant_group_shape = (
layer.query_quant.group_shape
if layer.query_quant is not None
else GroupShape.PER_TENSOR
)
self.q_size = self.num_heads * self.head_size
self.k_size = self.num_kv_heads * self.head_size
self.v_size = self.num_kv_heads * self.head_size_v
self.rope_matcher = MatcherRotaryEmbedding(
is_neox=is_neox,
head_size=self.head_size,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
self.rope_matcher: MatcherRotaryEmbedding | MatcherMRotaryEmbedding = (
MatcherRotaryEmbedding(
is_neox=is_neox,
head_size=self.head_size,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
)
)
def get_inputs(self) -> list[torch.Tensor]:
def get_inputs(self) -> list:
T = 5
L = 4096
qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size)
@@ -166,8 +252,13 @@ class QkNormRopeKvCachePattern:
if self.quant_query:
q_scale = empty_fp32(1)
inputs += [q_scale]
if _USE_LAYERNAME:
inputs.append(self.encoded_layer_name)
return inputs
def _get_layer_name(self, layer_name: LayerNameType | None) -> LayerNameType:
return self.encoded_layer_name if layer_name is None else layer_name
def pattern_non_fp8_quant_query(
self,
qkv: torch.Tensor,
@@ -175,6 +266,7 @@ class QkNormRopeKvCachePattern:
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
layer_name: LayerNameType | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
@@ -190,7 +282,9 @@ class QkNormRopeKvCachePattern:
q_rope = q_rope.view(-1, self.num_heads, self.head_size)
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size_v)
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
dummy = torch.ops.vllm.unified_kv_cache_update(
k_rope, v, self._get_layer_name(layer_name)
)
return dummy, q_rope, k_rope, v
def replacement_non_fp8_quant_query(
@@ -200,6 +294,7 @@ class QkNormRopeKvCachePattern:
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
layer_name: LayerNameType | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q_out = torch.empty(
qkv.shape[0],
@@ -228,10 +323,49 @@ class QkNormRopeKvCachePattern:
rms_norm_eps=self.eps,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
layer_name=self.layer_name,
layer_name=self._get_layer_name(layer_name),
)
return results[0], results[1], results[2], v
def _quantize_query(
self,
query: torch.Tensor,
q_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if self.query_quant_group_shape.is_per_tensor():
q_out = torch.empty_like(query, dtype=current_platform.fp8_dtype())
q_quant = auto_functionalized(
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
out=q_out,
x=query,
scale=q_scale,
is_dynamic=False,
)
# The AITER op's schema marks scale mutable, so preserve its
# functionalized write-back as an output of the matched region.
return q_quant[1], q_quant[2]
# QuantFP8.forward_hip falls back to the generic static op for the
# historical per-head shape (-1, block_size). This also covers the
# degenerate one-local-KV-head case where a scalar scale is represented
# as (-1, q_size), rather than changing Attention's classification.
q_out = torch.empty(
query.shape,
device=query.device,
dtype=current_platform.fp8_dtype(),
)
q_quant = auto_functionalized(
torch.ops._C.static_scaled_fp8_quant.default,
result=q_out,
input=query,
scale=q_scale,
group_shape=[
self.query_quant_group_shape.row,
self.query_quant_group_shape.col,
],
)
return q_quant[1], None
def pattern_fp8_quant_query(
self,
qkv: torch.Tensor,
@@ -240,7 +374,8 @@ class QkNormRopeKvCachePattern:
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
layer_name: LayerNameType | None = None,
) -> tuple[torch.Tensor, ...]:
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps)
@@ -251,24 +386,15 @@ class QkNormRopeKvCachePattern:
k_flat = k_normed.view(-1, self.k_size)
q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache)
# Match the quant-query op Attention.forward inserts (fp8 KV + UNIFIED).
# Explicit auto_functionalized (out=[1]) keeps the quant node in the pattern.
q_out = torch.empty_like(q_rope, dtype=current_platform.fp8_dtype())
q_quant = auto_functionalized(
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
out=q_out,
x=q_rope,
scale=q_scale,
is_dynamic=False,
)
# `scale` is mutable: its copy_ write-back to _q_scale bumps the mutation
# region, so keep q flat (a reshape lands past the barrier and won't match).
q_rope_fp8 = q_quant[1]
q_scale_out = q_quant[2]
q_rope_fp8, q_scale_out = self._quantize_query(q_rope, q_scale)
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size_v)
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
dummy = torch.ops.vllm.unified_kv_cache_update(
k_rope, v, self._get_layer_name(layer_name)
)
if q_scale_out is None:
return dummy, q_rope_fp8, k_rope, v
return dummy, q_rope_fp8, k_rope, v, q_scale_out
def replacement_fp8_quant_query(
@@ -279,7 +405,8 @@ class QkNormRopeKvCachePattern:
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
layer_name: LayerNameType | None = None,
) -> tuple[torch.Tensor, ...]:
q_out = torch.empty(
qkv.shape[0],
self.num_heads,
@@ -307,22 +434,14 @@ class QkNormRopeKvCachePattern:
rms_norm_eps=self.eps,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
layer_name=self.layer_name,
layer_name=self._get_layer_name(layer_name),
)
# Re-apply the quant on the kernel's bf16 q_out; fused op does not quant q.
# Same explicit auto_functionalized form as the pattern: [1] = quantized
# q, [2] = scale (returned so the buffer-writeback use is preserved).
# Re-apply the same quant form on the kernel's bf16 q_out; the fused
# operation does not quantize Q.
q_fp8_flat = results[1].view(-1, self.q_size)
q_fp8_out = torch.empty_like(q_fp8_flat, dtype=current_platform.fp8_dtype())
q_requant = auto_functionalized(
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
out=q_fp8_out,
x=q_fp8_flat,
scale=q_scale,
is_dynamic=False,
)
q_fp8 = q_requant[1] # flat to mirror the pattern (see note above)
q_scale_out = q_requant[2]
q_fp8, q_scale_out = self._quantize_query(q_fp8_flat, q_scale)
if q_scale_out is None:
return results[0], q_fp8, results[2], v
return results[0], q_fp8, results[2], v, q_scale_out
@staticmethod
@@ -345,6 +464,9 @@ class QkNormRopeKvCachePattern:
view_to_reshape(gm)
def _extra_check(self, match: pm.Match) -> bool:
return True
def _register(self, pattern, replacement, pm_pass) -> None:
trace_fn = QkNormRopeKvCachePattern.wrap_trace_fn(
pm.fwd_only,
@@ -370,40 +492,338 @@ class QkNormRopeKvCachePattern:
inputs,
trace_fn,
pm_pass,
extra_check=self._extra_check,
search_fn_pattern=search_fn_pattern,
)
def register(self, pm_pass: PatternMatcherPass) -> None:
# make_fx counts `self` in bound-method code params; wrap as plain fns.
# Distinct names per branch so mypy doesn't see one name, two signatures.
if self.quant_query:
if _USE_LAYERNAME:
def pattern_q(qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale):
return self.pattern_fp8_quant_query(
def pattern_q_with_layer(
qkv,
positions,
q_weight,
k_weight,
cos_sin_cache,
q_scale,
layer_name,
):
return self.pattern_fp8_quant_query(
qkv,
positions,
q_weight,
k_weight,
cos_sin_cache,
q_scale,
layer_name,
)
def replacement_q_with_layer(
qkv,
positions,
q_weight,
k_weight,
cos_sin_cache,
q_scale,
layer_name,
):
return self.replacement_fp8_quant_query(
qkv,
positions,
q_weight,
k_weight,
cos_sin_cache,
q_scale,
layer_name,
)
self._register(pattern_q_with_layer, replacement_q_with_layer, pm_pass)
else:
def pattern_q_without_layer(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
)
):
return self.pattern_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
)
def replacement_q(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
):
return self.replacement_fp8_quant_query(
def replacement_q_without_layer(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
)
):
return self.replacement_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
)
self._register(pattern_q, replacement_q, pm_pass)
self._register(
pattern_q_without_layer, replacement_q_without_layer, pm_pass
)
else:
if _USE_LAYERNAME:
def pattern_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
return self.pattern_non_fp8_quant_query(
def pattern_noq_with_layer(
qkv,
positions,
q_weight,
k_weight,
cos_sin_cache,
layer_name,
):
return self.pattern_non_fp8_quant_query(
qkv,
positions,
q_weight,
k_weight,
cos_sin_cache,
layer_name,
)
def replacement_noq_with_layer(
qkv,
positions,
q_weight,
k_weight,
cos_sin_cache,
layer_name,
):
return self.replacement_non_fp8_quant_query(
qkv,
positions,
q_weight,
k_weight,
cos_sin_cache,
layer_name,
)
self._register(
pattern_noq_with_layer, replacement_noq_with_layer, pm_pass
)
else:
def pattern_noq_without_layer(
qkv, positions, q_weight, k_weight, cos_sin_cache
):
return self.pattern_non_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache
)
def replacement_noq_without_layer(
qkv, positions, q_weight, k_weight, cos_sin_cache
):
return self.replacement_non_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache
)
self._register(
pattern_noq_without_layer,
replacement_noq_without_layer,
pm_pass,
)
def replacement_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
return self.replacement_non_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache
)
self._register(pattern_noq, replacement_noq, pm_pass)
class QkNormMRopeKvCachePattern(QkNormRopeKvCachePattern):
FUSED_OP = torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default
def __init__(
self,
layer: Attention,
eps: float,
is_neox: bool,
quant_query: bool,
rotary_dim: int,
mrope_section: tuple[int, int, int],
is_interleaved: bool,
) -> None:
super().__init__(
layer=layer,
eps=eps,
is_neox=is_neox,
quant_query=quant_query,
)
self.rotary_dim = rotary_dim
self.mrope_section = mrope_section
self.is_interleaved = is_interleaved
self.rope_matcher = MatcherMRotaryEmbedding(
is_neox=is_neox,
head_size=self.head_size,
rotary_dim=rotary_dim,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
mrope_section=mrope_section,
mrope_interleaved=is_interleaved,
)
def get_inputs(self) -> list[torch.Tensor]:
T = 5
L = 4096
qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size)
positions = empty_i64(3, T)
q_weight = empty_bf16(1, self.head_size)
k_weight = empty_bf16(1, self.head_size)
cos_sin_cache = empty_bf16(L, self.rotary_dim)
inputs = [qkv, positions, q_weight, k_weight, cos_sin_cache]
if self.quant_query:
inputs.append(empty_fp32(1))
if _USE_LAYERNAME:
inputs.append(self.encoded_layer_name)
return inputs
def _unfused_kv_views(self, qkv: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
# The supported backend reads K/V from the cache after the fused write.
# Keep view-compatible placeholders for the attention custom-op ABI.
_, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
k = k.view(qkv.shape[0], self.num_kv_heads, self.head_size)
v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v)
return k, v
def _fused_results(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
layer_name: LayerNameType | None,
) -> tuple[torch.Tensor, ...]:
q_out = torch.empty(
qkv.shape[0],
self.num_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
return auto_functionalized(
self.FUSED_OP,
q_out=q_out,
qkv=qkv,
positions=positions,
q_weight=q_weight,
k_weight=k_weight,
rms_norm_eps=self.eps,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
mrope_section_t=self.mrope_section[0],
mrope_section_h=self.mrope_section[1],
mrope_section_w=self.mrope_section[2],
is_interleaved=self.is_interleaved,
rotary_dim=self.rotary_dim,
layer_name=self._get_layer_name(layer_name),
)
def replacement_non_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
layer_name: LayerNameType | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
k, v = self._unfused_kv_views(qkv)
results = self._fused_results(
qkv, positions, q_weight, k_weight, cos_sin_cache, layer_name
)
return results[0], results[1], k, v
def replacement_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_scale: torch.Tensor,
layer_name: LayerNameType | None = None,
) -> tuple[torch.Tensor, ...]:
k, v = self._unfused_kv_views(qkv)
results = self._fused_results(
qkv, positions, q_weight, k_weight, cos_sin_cache, layer_name
)
q_flat = results[1].view(-1, self.q_size)
q_fp8, q_scale_out = self._quantize_query(q_flat, q_scale)
if q_scale_out is None:
return results[0], q_fp8, k, v
return results[0], q_fp8, k, v, q_scale_out
def _extra_check(self, match: pm.Match) -> bool:
expected = {
"head_size": self.head_size,
"rotary_dim": self.rotary_dim,
"mrope_section_t": self.mrope_section[0],
"mrope_section_h": self.mrope_section[1],
"mrope_section_w": self.mrope_section[2],
"mrope_interleaved": self.is_interleaved,
"is_neox_style": self.is_neox,
}
mrope_node = next(
(node for node in match.nodes if node.args and node.args[0] == MROPE_OP),
None,
)
if mrope_node is None or any(
mrope_node.kwargs.get(name) != value for name, value in expected.items()
):
return False
# The fused AITER MRoPE entrypoint writes rotated K directly to the
# cache and does not expose a BF16 K result. Returning the raw K view is
# valid only for decoder unified attention, whose backend consumes K
# from that cache. Reject graphs that observe K in any other way.
output_nodes = match.output_nodes()
if len(output_nodes) < 3 or output_nodes[2] is None:
return False
matched_nodes = set(match.nodes)
attention_op = torch.ops.vllm.unified_attention_with_output.default
def is_unified_attention_user(user: fx.Node) -> bool:
return user.op == "call_function" and (
user.target == attention_op
or (user.args and user.args[0] == attention_op)
)
return all(
user in matched_nodes or is_unified_attention_user(user)
for user in output_nodes[2].users
)
def _discover_mrope_configs(
config: VllmConfig,
) -> tuple[tuple[tuple[int, int, int], bool], ...]:
model_config = config.model_config
hf_configs = (
getattr(model_config, "hf_text_config", None),
getattr(model_config, "hf_config", None),
)
configs: set[tuple[tuple[int, int, int], bool]] = set()
for hf_config in hf_configs:
if hf_config is None:
continue
rope_parameters = getattr(hf_config, "rope_parameters", None)
if rope_parameters is None:
rope_parameters = getattr(hf_config, "rope_scaling", None)
if not isinstance(rope_parameters, Mapping):
continue
section = rope_parameters.get("mrope_section")
if not isinstance(section, (list, tuple)) or len(section) != 3:
continue
try:
parsed_section = (
int(section[0]),
int(section[1]),
int(section[2]),
)
except (TypeError, ValueError):
continue
if any(value <= 0 for value in parsed_section):
continue
configs.add(
(
parsed_section,
bool(rope_parameters.get("mrope_interleaved", False)),
)
)
return tuple(sorted(configs))
# ---------------------------------------------------------------------------
@@ -412,7 +832,7 @@ class QkNormRopeKvCachePattern:
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
"""Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
"""Fuse QK-norm + RoPE/MRoPE + KV cache update into an AITER HIP kernel.
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
attention layers that support the combined operation, eliminating two
@@ -429,22 +849,33 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
cc = config.compilation_config
self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num
self.mrope_configs = (
_discover_mrope_configs(config)
if cc.is_custom_op_enabled("rotary_embedding")
else ()
)
dtype = config.model_config.dtype
if dtype not in (torch.bfloat16, torch.float16):
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled: unsupported dtype %s", dtype
"QK Norm+RoPE/MRoPE+KVCache fusion not enabled: unsupported dtype %s",
dtype,
)
return
attn_layers = get_layers_from_vllm_config(config, Attention)
for _, layer in attn_layers.items():
if not layer.impl.fused_qk_norm_rope_kvcache_supported():
supports_rope = layer.impl.fused_qk_norm_rope_kvcache_supported()
supports_mrope = (
bool(self.mrope_configs)
and layer.impl.fused_qk_norm_mrope_kvcache_supported()
)
if not supports_rope and not supports_mrope:
continue
if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS:
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
"QK Norm+RoPE/MRoPE+KVCache fusion not enabled for a layer: "
"head_size=%d is not supported by the "
"fused_qk_norm_rope_cache_pts_quant_shuffle kernel "
"(supported: %s). Falling back to the unfused path.",
@@ -455,22 +886,71 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
if layer.head_size_v != layer.head_size:
# The fused kernel uses a single head_dim for q/k/v.
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
"QK Norm+RoPE/MRoPE+KVCache fusion not enabled for a layer: "
"head_size_v=%d differs from head_size=%d, which the fused "
"kernel does not support. Falling back to the unfused path.",
layer.head_size_v,
layer.head_size,
)
continue
for epsilon in [1e-5, 1e-6]:
for neox in [True, False]:
for quant_q in [False, True]:
QkNormRopeKvCachePattern(
layer=layer,
eps=epsilon,
is_neox=neox,
quant_query=quant_q,
).register(self.patterns)
if supports_rope:
for epsilon in [1e-5, 1e-6]:
for neox in [True, False]:
for quant_q in [False, True]:
QkNormRopeKvCachePattern(
layer=layer,
eps=epsilon,
is_neox=neox,
quant_query=quant_q,
).register(self.patterns)
if supports_mrope:
for section, is_interleaved in self.mrope_configs:
rotary_dim = 2 * sum(section)
if rotary_dim > layer.head_size:
logger.warning_once(
"QK Norm+MRoPE+KVCache fusion not enabled for a "
"layer: rotary_dim=%d exceeds head_size=%d.",
rotary_dim,
layer.head_size,
)
continue
vector_width = layer.head_size // 32
for epsilon in [1e-5, 1e-6]:
for neox in [True, False]:
# Each warp lane processes `vector_width` adjacent
# values. NeoX additionally exchanges vectors
# across the half-rotary boundary.
rotary_alignment = vector_width * (2 if neox else 1)
if rotary_dim % rotary_alignment != 0:
logger.warning_once(
"QK Norm+MRoPE+KVCache fusion not enabled "
"for a layer/style: rotary_dim=%d must be "
"divisible by %d for head_size=%d and "
"is_neox=%s.",
rotary_dim,
rotary_alignment,
layer.head_size,
neox,
)
continue
for quant_q in [False, True]:
QkNormMRopeKvCachePattern(
layer=layer,
eps=epsilon,
is_neox=neox,
quant_query=quant_q,
rotary_dim=rotary_dim,
mrope_section=section,
is_interleaved=is_interleaved,
).register(self.patterns)
# Opaque LayerName is a pattern wildcard on torch >= 2.11, so
# homogeneous attention layers produce duplicate search patterns.
# As in the other LayerName-aware fusion passes, one supported
# layer registers all shape/style variants for the model.
if _USE_LAYERNAME:
break
self.dump_patterns(config, self.patterns)
@@ -478,8 +958,8 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
def __call__(self, graph: fx.Graph) -> None:
self.matched_count = self.patterns.apply(graph)
logger.info(
"QK-Norm+RoPE+KVCache fusion: replaced %s pattern(s) "
"with AITER fused_qk_norm_rope_cache_pts_quant_shuffle",
"QK-Norm+RoPE/MRoPE+KVCache fusion: replaced %s pattern(s) "
"with an AITER fused attention-prologue kernel",
self.matched_count,
)
@@ -487,4 +967,10 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
return compile_range.end <= self.max_token_num
def uuid(self) -> str:
return VllmInductorPass.hash_source(self, QkNormRopeKvCachePattern)
return VllmInductorPass.hash_source(
self,
QkNormRopeKvCachePattern,
QkNormMRopeKvCachePattern,
_discover_mrope_configs,
repr(self.mrope_configs),
)
+4 -4
View File
@@ -154,15 +154,15 @@ class PassConfig:
fuse_rope_kvcache: bool = None # type: ignore[assignment]
"""Fuse the QK rope + KV cache ops."""
fuse_qk_norm_rope_kvcache: bool = Field(default=None) # type: ignore[assignment]
"""Fuse QK RMSNorm + RoPE + KV cache update into a single AITER HIP
"""Fuse QK RMSNorm + RoPE/MRoPE + KV cache update into an AITER HIP
kernel. Supersedes both enable_qk_norm_rope_fusion and fuse_rope_kvcache
for layers that support it. Auto-enabled at O1+ on ROCm for models
with QK-norm (e.g. Qwen3-MoE)."""
for layers that support it. Auto-enabled at O2+ on ROCm for models
with QK-norm (e.g. Qwen3-MoE and Qwen3-VL-class architectures)."""
rope_kvcache_fusion_max_token_num: int = 256
"""The threshold for ROCm AITER RoPE+KVCache fusion e.g. for small batch decode.
Larger batch sizes e.g. during prefill will use the unfused kernels.
Also applies to the fused QK-Norm+RoPE+KVCache pass.
Also applies to the fused QK-Norm+RoPE/MRoPE+KVCache pass.
"""
fi_allreduce_fusion_max_size_mb: float | None = None
+50 -1
View File
@@ -242,7 +242,7 @@ def enable_mla_dual_rms_norm_fusion(cfg: "VllmConfig") -> bool:
def enable_qk_norm_rope_kvcache(cfg: "VllmConfig") -> bool:
"""Enable fused QK-norm + RoPE + KV cache update on ROCm with AITER."""
"""Enable fused QK-norm + RoPE/MRoPE + KV cache update with AITER."""
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_enabled():
@@ -776,6 +776,45 @@ class VllmConfig:
architectures = set(model_config.architectures)
return bool(architectures & default_breakable_cudagraph_architectures())
def _uses_breakable_cudagraph_for_batch_invariance(self) -> bool:
"""Avoid freezing runtime-M tile lookup in compiled forward (#54243).
Breakable graphs look up tuned bf16, unquantized qkv/o/gate_up/down tiles
at capture; lm_head runs outside compiled forward and does not benefit."""
from vllm.model_executor.determinism import batch_invariant_configs as bi
from vllm.platforms import current_platform
model = self.model_config
if (
not envs.VLLM_BATCH_INVARIANT
or model is None
or model.enforce_eager
or model.dtype != torch.bfloat16
or model.quantization is not None
or not current_platform.is_cuda()
# Sequence parallelism / async TP are torch.compile passes.
or self.compilation_config.pass_config.enable_sp
or self.compilation_config.pass_config.fuse_gemm_comms
):
return False
family = bi._get_tuned_matmul_arch_family(
current_platform.get_device_capability()
)
if family is None or family not in bi._BATCH_INVARIANT_MATMUL_TUNED_CONFIGS:
return False
table = bi._BATCH_INVARIANT_MATMUL_TUNED_CONFIGS[family]
parallel = self.parallel_config
tp = parallel.tensor_parallel_size
hidden = model.get_hidden_size()
head = model.get_head_size()
heads = model.get_num_attention_heads(parallel)
kv_heads = model.get_num_kv_heads(parallel)
shapes = [((heads + 2 * kv_heads) * head, hidden), (hidden, heads * head)]
intermediate = getattr(model.hf_text_config, "intermediate_size", None)
# Per-layer sizes (e.g. Gemma3n) are not modeled.
if isinstance(intermediate, int):
shapes += [(2 * intermediate // tp, hidden), (hidden, intermediate // tp)]
return any(shape in table for shape in shapes)
def _maybe_enable_breakable_cudagraph(self) -> bool:
if (
"VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ
@@ -786,6 +825,16 @@ class VllmConfig:
"Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. "
"Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out."
)
elif (
"VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ
and envs.VLLM_BATCH_INVARIANT
and self._uses_breakable_cudagraph_for_batch_invariance()
):
os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
logger.info_once(
"VLLM_BATCH_INVARIANT=1: auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. "
"Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out."
)
from vllm.compilation.breakable_cudagraph import (
is_breakable_cudagraph_enabled,
@@ -103,6 +103,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# (``_pop_done_transfers``); guarded by
# ``_sending_transfers_lock``.
self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list)
self._send_failures: set[ReqId] = set()
self._sending_transfers_lock = threading.Lock()
# Writer-thread owned matching state.
@@ -162,6 +163,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
for handle in handles:
self.nixl_wrapper.release_xfer_handle(handle)
self._sending_transfers.clear()
self._send_failures.clear()
super().shutdown()
# --- Engine-main-thread entry point -------------------------------- #
@@ -728,6 +730,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# don't have a ``_recving_metadata`` entry to invalidate, so
# we just release the handle and let the engine reschedule
# via the lease / watchdog.
with self._sending_transfers_lock:
self._send_failures.add(request_id)
if not self._handle_failed_transfer(request_id, handle):
return handle
return None
@@ -811,13 +815,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
done_pushing, failed_pushing = self._pop_done_transfers(
self._sending_transfers
)
# A failed send must never be reported as done: its blocks
# are freed via the lease / watchdog instead.
done_pushing = {
req_id
for req_id in done_pushing - failed_pushing
if req_id in self._recving_metadata
}
# Remember failures until the final sibling WRITE completes.
self._send_failures.update(failed_pushing)
successful = {
req_id
for req_id in done_pushing - self._send_failures
if req_id in self._reqs_to_send or req_id in self._reqs_to_process
}
self._send_failures.difference_update(done_pushing | done_sending)
# Expired requests were already reported, even if their WRITEs finish later.
done_pushing = successful
for req_id in done_pushing:
self._reqs_to_send.pop(req_id, None)
self._reqs_to_process.discard(req_id)
@@ -116,9 +116,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
See https://platform.openai.com/docs/api-reference/completions/create
for the API specification. This API mimics the OpenAI Completion API.
NOTE: Currently we do not support the following feature:
- suffix (the language models we currently support do not support
suffix)
NOTE: suffix is only supported by models that implement FIM rendering.
"""
return await self._with_kv_transfer_rejection_cleanup(
self._create_completion(request, raw_request), request, raw_request
@@ -31,11 +31,14 @@ class HummingLinearKernel(MPLinearKernel):
)
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
if self.w_zp_name is not None and hasattr(layer, self.w_zp_name):
name_map["zero_point"] = self.w_zp_name
group_size = self.config.group_size
quant_config = {
"quant_method": "humming",
"dtype": "int" + str(self.config.weight_type.size_bits),
"group_size": 0 if group_size == -1 else group_size,
"has_zero_point": self.config.zero_points,
}
if self.config.zero_points:
@@ -337,6 +337,25 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
)
intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K))
# Under EP, drop the top-k slots routed to remote experts at alignment
# time instead of launching `off_experts == -1` GEMM blocks that only
# write zeros; the pad-aware `moe_sum` below skips the same slots.
# Their rows in the workspace caches are then never written, so every
# consumer of those caches must be row-local: a dynamic per-tensor
# scale for the second GEMM's activation would take its amax over the
# untouched rows, and LoRA aligns the full `topk_ids` itself.
a2_scale_is_global = (
self.quant_dtype is not None
and a2_scale is None
and self.block_shape is None
and (not self.per_act_token_quant or self.quantization_emulation)
)
skip_invalid = (
expert_map is not None
and self._lora_context is None
and not a2_scale_is_global
)
# Include fused shared-expert rows while preserving EP remapping.
num_align_experts = w1.shape[0] if expert_map is None else global_num_experts
sorted_token_ids, expert_ids, num_tokens_post_padded = (
@@ -350,6 +369,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
block_shape=self.block_shape,
ignore_invalid_experts=skip_invalid,
)
)
@@ -585,10 +605,21 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
)
# separate function is required for MoE + LoRA
self.moe_sum(intermediate_cache3, output)
self.moe_sum(intermediate_cache3, output, topk_ids, expert_map)
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
ops.moe_sum(input, output)
def moe_sum(
self,
input: torch.Tensor,
output: torch.Tensor,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
) -> None:
if expert_map is not None:
# Skip the slots whose expert is not on this rank: the rows the
# alignment dropped, or the zeros the `-1` blocks wrote.
ops.moe_sum(input, output, topk_ids, expert_map)
else:
ops.moe_sum(input, output)
class TritonWNA16Experts(TritonExperts):
@@ -41,6 +41,7 @@ if current_platform.is_cuda():
def _dsv4_topk_kernel(
gating_output_ptr,
correction_bias_ptr,
is_padding_ptr,
topk_weights_ptr,
topk_ids_ptr,
routed_scaling_factor,
@@ -49,6 +50,7 @@ if current_platform.is_cuda():
image_sentinel_lo,
NUM_EXPERTS: tl.constexpr,
BLOCK_N: tl.constexpr,
HAS_PADDING: tl.constexpr,
HAS_VL: tl.constexpr,
launch_pdl: tl.constexpr,
):
@@ -106,6 +108,11 @@ if current_platform.is_cuda():
output_mask = topk_offsets < 6
output_offsets = row * 6 + topk_offsets
if HAS_PADDING:
is_padding = tl.load(is_padding_ptr + row)
selected_weights = tl.where(is_padding, 0.0, selected_weights)
selected_ids = tl.where(is_padding, -1, selected_ids)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
@@ -118,11 +125,20 @@ def dsv4_topk(
correction_bias: torch.Tensor,
indices_dtype: torch.dtype,
routed_scaling_factor: float,
is_padding: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens, num_experts = gating_output.shape
if is_padding is not None:
assert is_padding.dtype == torch.bool
assert is_padding.shape == (num_tokens,)
assert is_padding.device == gating_output.device
assert is_padding.is_contiguous()
assert indices_dtype in (torch.int32, torch.int64), (
"Padding requires a signed indices dtype for the -1 sentinel."
)
has_vl = bias_vl is not None and image_sentinel_lo > 0
if bias_vl is not None:
assert input_ids is not None, "bias_vl routing requires input_ids"
@@ -136,6 +152,7 @@ def dsv4_topk(
_dsv4_topk_kernel[(num_tokens,)](
gating_output,
correction_bias,
is_padding,
topk_weights,
topk_ids,
routed_scaling_factor,
@@ -144,6 +161,7 @@ def dsv4_topk(
image_sentinel_lo,
NUM_EXPERTS=num_experts,
BLOCK_N=triton.next_power_of_2(num_experts),
HAS_PADDING=is_padding is not None,
HAS_VL=has_vl,
num_warps=1,
launch_pdl=current_platform.is_arch_support_pdl(),
@@ -148,9 +148,13 @@ def fused_topk_bias(
)
output_indices_dtype = torch.int32 if indices_type is None else indices_type
padding_mask = _get_padding_mask(gating_output.shape[0])
if (
scoring_func == "sqrtsoftplus"
and hash_indices_table is None
# dsv4_topk marks padded rows with a -1 sentinel, which unsigned
# indices cannot represent; keep the generic path for that pair.
and (padding_mask is None or output_indices_dtype != torch.uint32)
and can_use_dsv4_topk(
gating_output,
e_score_correction_bias,
@@ -165,6 +169,7 @@ def fused_topk_bias(
e_score_correction_bias,
output_indices_dtype,
routed_scaling_factor,
is_padding=padding_mask,
input_ids=input_tokens,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
@@ -787,6 +787,7 @@ class CompressedTensorsConfig(QuantizationConfig):
num_bits=weight_quant.num_bits,
strategy=weight_quant.strategy,
group_size=weight_quant.group_size,
symmetric=weight_quant.symmetric,
input_quant=input_quant,
layer_name=layer_name,
quant_format=format,
@@ -799,6 +800,7 @@ class CompressedTensorsConfig(QuantizationConfig):
num_bits=weight_quant.num_bits,
strategy=weight_quant.strategy,
group_size=weight_quant.group_size,
symmetric=weight_quant.symmetric,
input_quant=input_quant,
layer_name=layer_name,
quant_format=format,
@@ -42,7 +42,15 @@ WNA16_SUPPORTED_TYPES_MAP = {
7: scalar_types.uint7b64,
8: scalar_types.uint8b128,
}
WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8}
WNA16_ZP_SUPPORTED_TYPES_MAP = {
2: scalar_types.uint2,
3: scalar_types.uint3,
4: scalar_types.uint4,
5: scalar_types.uint5,
6: scalar_types.uint6,
7: scalar_types.uint7,
8: scalar_types.uint8,
}
WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming.
"""Weight N-bit INT scheme with INT4 activation quant via Humming.
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
with INT4 symmetric dynamic per-token/per-group input activation
quantization. Static, per-tensor, and asymmetric activation quantization
are not supported.
quantization. Supports both symmetric and asymmetric weight quantization.
Static, per-tensor, and asymmetric activation quantization are not supported.
"""
import math
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
WNA16_SUPPORTED_TYPES_MAP,
WNA16_ZP_SUPPORTED_TYPES_MAP,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_repeat_scales_on_all_ranks,
@@ -36,6 +37,7 @@ from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
GroupQuantScaleParameter,
PackedColumnParameter,
PackedvLLMParameter,
)
@@ -52,6 +54,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
num_bits: int,
strategy: str,
group_size: int | None = None,
symmetric: bool = True,
input_quant: QuantizationArgs | None = None,
layer_name: str | None = None,
quant_format: str = "pack-quantized",
@@ -60,6 +63,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
self.pack_factor = Fraction(32, num_bits)
self.strategy = strategy
self.group_size = -1 if group_size is None else group_size
self.symmetric = symmetric
self.input_quant = input_quant
self.layer_name = layer_name
self.quant_format = quant_format
@@ -69,7 +73,19 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
f"Unsupported num_bits = {num_bits} for WNA4Int; "
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
raise ValueError(
f"Asymmetric quantization not supported for "
f"num_bits = {num_bits}. Supported: "
f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = (
WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits]
if not self.symmetric
else WNA16_SUPPORTED_TYPES_MAP[num_bits]
)
if input_quant is not None:
if not input_quant.symmetric:
@@ -143,7 +159,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
weight_type=self.quant_type,
act_type=params_dtype,
group_size=self.group_size,
zero_points=False,
zero_points=not self.symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_config)
@@ -151,12 +167,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__)
self._kernel_backends_being_used.add(kernel_type.__name__)
self.kernel = kernel_type(
mp_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
)
input_quant_config = self._build_input_quant_config()
if input_quant_config is not None:
layer._humming_input_quant_config = input_quant_config
@@ -167,10 +177,10 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
partition_scales = not marlin_repeat_scales_on_all_ranks(
self.group_size, row_parallel
)
scales_size = input_size // group_size
scales_and_zp_size = input_size // group_size
if partition_scales:
assert input_size_per_partition % group_size == 0
scales_size = input_size_per_partition // group_size
scales_and_zp_size = input_size_per_partition // group_size
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
layer.register_parameter(
@@ -190,7 +200,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
)
scale_data = torch.empty(
output_size_per_partition, scales_size, dtype=params_dtype
output_size_per_partition, scales_and_zp_size, dtype=params_dtype
)
if partition_scales:
weight_scale = GroupQuantScaleParameter(
@@ -207,6 +217,34 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
)
layer.register_parameter("weight_scale", weight_scale)
if not self.symmetric:
packed_output_dim = math.ceil(
output_size_per_partition * self.num_bits / 32
)
zeros_data = torch.zeros(
packed_output_dim,
scales_and_zp_size,
dtype=torch.int32,
)
if not partition_scales:
qzeros = PackedColumnParameter(
output_dim=0,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
data=zeros_data,
)
else:
qzeros = PackedvLLMParameter(
input_dim=1,
output_dim=0,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
data=zeros_data,
)
layer.register_parameter("weight_zero_point", qzeros)
layer.register_parameter(
"weight_shape",
BasevLLMParameter(
@@ -215,6 +253,13 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
),
)
self.kernel = kernel_type(
mp_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming.
"""Weight N-bit INT scheme with INT8 activation quant via Humming.
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
with INT8 symmetric dynamic per-token/per-group input activation
quantization. Static, per-tensor, and asymmetric activation quantization
are not supported.
quantization. Supports both symmetric and asymmetric weight quantization.
Static, per-tensor, and asymmetric activation quantization are not supported.
"""
import math
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
WNA16_SUPPORTED_TYPES_MAP,
WNA16_ZP_SUPPORTED_TYPES_MAP,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_repeat_scales_on_all_ranks,
@@ -36,6 +37,7 @@ from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
GroupQuantScaleParameter,
PackedColumnParameter,
PackedvLLMParameter,
)
@@ -52,6 +54,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
num_bits: int,
strategy: str,
group_size: int | None = None,
symmetric: bool = True,
input_quant: QuantizationArgs | None = None,
layer_name: str | None = None,
quant_format: str = "pack-quantized",
@@ -60,6 +63,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
self.pack_factor = Fraction(32, num_bits)
self.strategy = strategy
self.group_size = -1 if group_size is None else group_size
self.symmetric = symmetric
self.input_quant = input_quant
self.layer_name = layer_name
self.quant_format = quant_format
@@ -69,7 +73,19 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
f"Unsupported num_bits = {num_bits} for WNA8Int; "
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
raise ValueError(
f"Asymmetric quantization not supported for "
f"num_bits = {num_bits}. Supported: "
f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
)
self.quant_type = (
WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits]
if not self.symmetric
else WNA16_SUPPORTED_TYPES_MAP[num_bits]
)
if input_quant is not None:
if not input_quant.symmetric:
@@ -143,7 +159,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
weight_type=self.quant_type,
act_type=params_dtype,
group_size=self.group_size,
zero_points=False,
zero_points=not self.symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_config)
@@ -151,12 +167,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__)
self._kernel_backends_being_used.add(kernel_type.__name__)
self.kernel = kernel_type(
mp_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
)
input_quant_config = self._build_input_quant_config()
if input_quant_config is not None:
layer._humming_input_quant_config = input_quant_config
@@ -167,10 +177,10 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
partition_scales = not marlin_repeat_scales_on_all_ranks(
self.group_size, row_parallel
)
scales_size = input_size // group_size
scales_and_zp_size = input_size // group_size
if partition_scales:
assert input_size_per_partition % group_size == 0
scales_size = input_size_per_partition // group_size
scales_and_zp_size = input_size_per_partition // group_size
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
layer.register_parameter(
@@ -190,7 +200,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
)
scale_data = torch.empty(
output_size_per_partition, scales_size, dtype=params_dtype
output_size_per_partition, scales_and_zp_size, dtype=params_dtype
)
if partition_scales:
weight_scale = GroupQuantScaleParameter(
@@ -207,6 +217,34 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
)
layer.register_parameter("weight_scale", weight_scale)
if not self.symmetric:
packed_output_dim = math.ceil(
output_size_per_partition * self.num_bits / 32
)
zeros_data = torch.zeros(
packed_output_dim,
scales_and_zp_size,
dtype=torch.int32,
)
if not partition_scales:
qzeros = PackedColumnParameter(
output_dim=0,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
data=zeros_data,
)
else:
qzeros = PackedvLLMParameter(
input_dim=1,
output_dim=0,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
data=zeros_data,
)
layer.register_parameter("weight_zero_point", qzeros)
layer.register_parameter(
"weight_shape",
BasevLLMParameter(
@@ -215,6 +253,13 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
),
)
self.kernel = kernel_type(
mp_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
@@ -5,7 +5,6 @@ import fnmatch
from typing import TYPE_CHECKING, Any, NamedTuple, cast
import torch
from transformers import PretrainedConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
@@ -82,10 +81,6 @@ __all__ = ["QuarkLinearMethod"]
logger = init_logger(__name__)
# model_type values that use dynamic MXFP4 re-quantization for
# OCP MX fp4 Quark checkpoints
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"})
class QuantKeyMatch(NamedTuple):
matches: bool
@@ -119,37 +114,6 @@ class QuarkConfig(QuantizationConfig):
self.kv_cache_group = kv_cache_group
self.kv_cache_config = kv_cache_config
self.pack_method = pack_method
# Note : this flag is kept disabled because the overhead of
# dynamic mxfp4 quantization negates the performance gains
# that come from shifting to mxfp4. It is left here in case
# we want to re-enable it in the future.
self.dynamic_mxfp4_quant = False
def maybe_update_config(
self,
model_name: str,
hf_config: PretrainedConfig | None = None,
revision: str | None = None,
):
"""Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints."""
if hf_config is None:
return
if (
getattr(hf_config, "model_type", None)
not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
):
return
quant_config = getattr(hf_config, "quantization_config", None)
if isinstance(quant_config, dict):
quant_dtype = (
quant_config.get("global_quant_config", {})
.get("weight", {})
.get("dtype")
)
if quant_dtype == "fp4":
self.dynamic_mxfp4_quant = True
def get_linear_method(self) -> "QuarkLinearMethod":
return QuarkLinearMethod(self)
@@ -201,17 +165,6 @@ class QuarkConfig(QuantizationConfig):
self.get_quant_method_target(prefix, type(layer))
)
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
is_ignored = should_ignore_layer(
prefix,
ignore=exclude_layers,
fused_mapping=self.packed_modules_mapping,
check_children=isinstance(layer, RoutedExperts),
)
dynamic_mxfp4_quant = (
is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant
)
if method_cls is UnquantizedFusedMoEMethod:
return UnquantizedFusedMoEMethod(layer.moe_config)
if method_cls is UnquantizedLinearMethod:
@@ -222,7 +175,6 @@ class QuarkConfig(QuantizationConfig):
scheme_cls,
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
weight_config=self._find_matched_config(prefix, type(layer)).get(
"weight"
),
@@ -274,11 +226,6 @@ class QuarkConfig(QuantizationConfig):
if is_routed_experts:
return None, None, UnquantizedFusedMoEMethod
if issubclass(layer_type, LinearBase):
if "self_attn" in prefix and self.dynamic_mxfp4_quant:
weight_quant_key, activation_key, _ = self.get_scheme_cls(
layer_type, prefix
)
return weight_quant_key, activation_key, QuarkLinearMethod
return None, None, UnquantizedLinearMethod
return None, None, None
if issubclass(layer_type, LinearBase):
@@ -956,7 +903,6 @@ class QuarkConfig(QuantizationConfig):
scheme_cls: type["QuarkScheme"],
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
weight_config: dict[str, Any] | None = None,
) -> "QuarkScheme":
"""Construct a Quark scheme selected by get_scheme_cls."""
@@ -996,9 +942,6 @@ class QuarkConfig(QuantizationConfig):
):
kwargs["weight_quant_key"] = weight_quant_key
if scheme_cls is QuarkOCP_MX:
kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant
scheme = scheme_cls(**kwargs)
# Raise error if device does not support the scheme
@@ -26,10 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
PackedvLLMParameter,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from .quark_scheme import QuarkScheme
@@ -46,10 +44,9 @@ class QuarkOCP_MX(QuarkScheme):
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
):
super().__init__(weight_quant_key, activation_quant_key)
self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
self.weight_dtype = next(
dtype
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
@@ -109,21 +106,9 @@ class QuarkOCP_MX(QuarkScheme):
def get_min_capability(cls) -> int:
return 70
def process_dynamic_mxfp4_weights_after_loading(
self, layer: torch.nn.Module
) -> None:
from aiter.ops.triton.quant import dynamic_mxfp4_quant
w_q, w_s = dynamic_mxfp4_quant(layer.weight)
layer.weight_scale = torch.nn.Parameter(w_s, requires_grad=False)
layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
if self.dynamic_mxfp4_quant:
self.process_dynamic_mxfp4_weights_after_loading(layer)
self.ocp_mx_linear.process_weights_after_loading(layer)
def create_weights(
@@ -145,51 +130,36 @@ class QuarkOCP_MX(QuarkScheme):
"tensor-parallel sharding for this layer."
)
if self.dynamic_mxfp4_quant:
weight = ModelWeightParameter(
data=torch.empty(
sum(output_partition_sizes),
input_size_per_partition,
dtype=params_dtype,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.register_parameter("weight", weight)
set_weight_attrs(weight, kwargs)
else:
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
# WEIGHT
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
# WEIGHT SCALE
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
if self.weight_quant_key == kMxfp4Static:
self.ocp_mx_linear = init_mxfp4_linear_kernel(
@@ -101,6 +101,19 @@ def prepare_humming_linear_layer_config(
param_dtype=layer.params_dtype,
)
if (
weight_schema.has_zero_point
and input_schema.a_dtype is not None
and weight_schema.b_dtype.num_bits == input_schema.a_dtype.num_bits
):
raise ValueError(
"Humming does not support asymmetric (zero_point) weights "
"when weight and activation bit widths are equal "
f"(W{weight_schema.b_dtype.num_bits}"
f"A{input_schema.a_dtype.num_bits}). "
"Use symmetric weights or a different activation bit width."
)
tensors.update(input_tensors)
layer.weight_schema = weight_schema
input_schema = check_and_fallback_input_schema(
@@ -7,6 +7,7 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
from .base import RotaryEmbeddingBase
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding, yarn_get_mscale
@@ -240,6 +241,62 @@ def apply_interleaved_rope(x: torch.Tensor, mrope_section: list[int]) -> torch.T
return torch.where(is_width, x[2], result)
def _mrope_apply(
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
cos_sin_cache: torch.Tensor,
head_size: int,
rotary_dim: int,
mrope_section_t: int,
mrope_section_h: int,
mrope_section_w: int,
mrope_interleaved: bool,
is_neox_style: bool,
) -> None:
cos_sin = cos_sin_cache[positions]
cos, sin = cos_sin.chunk(2, dim=-1)
query_out, key_out = triton_mrope(
query,
key,
cos,
sin,
[mrope_section_t, mrope_section_h, mrope_section_w],
head_size,
rotary_dim,
mrope_interleaved,
is_neox_style,
)
if query_out.data_ptr() != query.data_ptr():
query.copy_(query_out.view_as(query))
if key_out.data_ptr() != key.data_ptr():
key.copy_(key_out.view_as(key))
def _mrope_apply_fake(
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
cos_sin_cache: torch.Tensor,
head_size: int,
rotary_dim: int,
mrope_section_t: int,
mrope_section_h: int,
mrope_section_w: int,
mrope_interleaved: bool,
is_neox_style: bool,
) -> None:
return
direct_register_custom_op(
op_name="mrope",
op_func=_mrope_apply,
mutates_args=["query", "key"],
fake_impl=_mrope_apply_fake,
)
class MRotaryEmbedding(RotaryEmbeddingBase):
"""Rotary Embedding with Multimodal Sections."""
@@ -426,6 +483,35 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
return query, key
def forward_hip(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
offsets: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if positions.ndim == 2:
assert key is not None
assert self.mrope_section
query_shape = query.shape
key_shape = key.shape
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
torch.ops.vllm.mrope(
positions,
query,
key,
cos_sin_cache,
self.head_size,
self.rotary_dim,
self.mrope_section[0],
self.mrope_section[1],
self.mrope_section[2],
self.mrope_interleaved,
self.is_neox_style,
)
return query.reshape(query_shape), key.reshape(key_shape)
return super().forward_hip(positions, query, key)
def forward_cpu(
self,
positions: torch.Tensor,
+3 -3
View File
@@ -498,11 +498,11 @@ class CpuPlatform(Platform):
return
# reconcile attention and mamba page sizes
backend_cls = cls._find_non_ssm_backend(vllm_config)
if backend_cls is None:
backend_classes = cls._find_non_ssm_backends(vllm_config)
if not backend_classes:
return
cls._align_hybrid_block_size(vllm_config, backend_cls)
cls._align_hybrid_block_size(vllm_config, backend_classes[0])
@classmethod
def discover_numa_topology(cls) -> list[list[int]]:
+72 -18
View File
@@ -3,6 +3,8 @@
import contextlib
import enum
import functools
import itertools
import math
import os
import platform
import sys
@@ -573,10 +575,10 @@ class Platform:
pass
@classmethod
def _find_non_ssm_backend(
def _find_non_ssm_backends(
cls, vllm_config: "VllmConfig"
) -> "type[AttentionBackend] | None":
"""Find the first non-SSM attention backend from model layers."""
) -> "list[type[AttentionBackend]]":
"""Distinct non-SSM attention backends, in layer order."""
from vllm.config.vllm import get_layers_from_vllm_config
from vllm.model_executor.layers.attention_layer_base import (
AttentionLayerBase,
@@ -586,11 +588,62 @@ class Platform:
vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
)
backends: list[type[AttentionBackend]] = []
for layer in attn_layers.values():
b = layer.get_attn_backend()
if not b.is_ssm():
return b
return None
if not b.is_ssm() and b not in backends:
backends.append(b)
return backends
@classmethod
def _preferred_block_size_for_backends(
cls,
backend_classes: "list[type[AttentionBackend]]",
default_block_size: int,
vllm_config: "VllmConfig",
) -> int:
"""Smallest block size every backend accepts.
``supports_block_size`` may be overridden to accept exact sizes only
(CPU_MLA takes 16 and no multiple of it), so candidates are the LCMs of
one supported size per backend, each checked against every backend.
Raises:
ValueError: If the backends share no supported block size.
"""
from vllm.config.vllm import set_current_vllm_config
from vllm.v1.attention.backend import MultipleOf
# Backends may read the current config to decide a preference.
with set_current_vllm_config(vllm_config):
if len(backend_classes) == 1:
return backend_classes[0].get_preferred_block_size(default_block_size)
if all(b.supports_block_size(default_block_size) for b in backend_classes):
return default_block_size
# A backend declaring no sizes accepts any, so it contributes 1.
per_backend_sizes = [
[
s.base if isinstance(s, MultipleOf) else s
for s in b.get_supported_kernel_block_sizes()
]
or [1]
for b in backend_classes
]
candidates = sorted(
{math.lcm(*sizes) for sizes in itertools.product(*per_backend_sizes)}
)
for candidate in candidates:
if all(b.supports_block_size(candidate) for b in backend_classes):
return candidate
raise ValueError(
"The attention backends share no supported KV cache block size ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
for b in backend_classes
)
+ ")."
)
@classmethod
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
@@ -598,7 +651,6 @@ class Platform:
For hybrid models, also aligns block_size with mamba page sizes.
"""
from vllm.config.cache import CacheConfig
from vllm.config.vllm import set_current_vllm_config
cache_config = vllm_config.cache_config
model_config = vllm_config.model_config
@@ -607,34 +659,36 @@ class Platform:
if not model_config:
return
backend_cls = cls._find_non_ssm_backend(vllm_config)
if backend_cls is None:
backend_classes = cls._find_non_ssm_backends(vllm_config)
if not backend_classes:
return
# Phase 1: Pick block size from backend (skip if user set --block-size)
# Phase 1: Pick a block size every attention backend supports (skip if
# user set --block-size). Models can mix backends with disjoint
# preferences, and a size from the first alone later fails
# select_common_block_size().
if not cache_config.user_specified_block_size:
with set_current_vllm_config(vllm_config):
preferred = backend_cls.get_preferred_block_size(
CacheConfig.DEFAULT_BLOCK_SIZE
)
preferred = cls._preferred_block_size_for_backends(
backend_classes, CacheConfig.DEFAULT_BLOCK_SIZE, vllm_config
)
if preferred != CacheConfig.DEFAULT_BLOCK_SIZE:
logger.info(
"Setting kv cache block size to %d for %s backend.",
"Setting kv cache block size to %d for %s backend(s).",
preferred,
backend_cls.get_name(),
"/".join(b.get_name() for b in backend_classes),
)
cache_config.block_size = preferred
# Phase 2: Align block/mamba sizes for hybrid models
# (may override user settings).
if model_config.is_hybrid:
cls._align_hybrid_block_size(vllm_config, backend_cls)
cls._align_hybrid_block_size(vllm_config, backend_classes[0])
# Phase 3: Align block/page sizes when multiple KV dtypes share the
# block pool (e.g. nvfp4 primary + unquantized skip layers).
# May override the user's --block-size.
if cache_config.kv_cache_dtype_skip_layers:
cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls)
cls._align_heterogeneous_kv_block_size(vllm_config, backend_classes[0])
@classmethod
def _align_heterogeneous_kv_block_size(
+20
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.metadata
import os
import platform
from datetime import timedelta
@@ -361,6 +362,25 @@ def get_cdna_version() -> int:
return 0
@cache
def get_rocm_version() -> tuple[int, ...] | None:
"""Return the installed ROCm release as (major, minor, patch), or None."""
# ROCm 10+ ships as the `rocm` pip SDK; older releases install to /opt/rocm.
try:
version = importlib.metadata.version("rocm")
except importlib.metadata.PackageNotFoundError:
rocm_path = os.environ.get("ROCM_PATH", "/opt/rocm")
try:
with open(os.path.join(rocm_path, ".info", "version")) as f:
version = f.read()
except OSError:
return None
match = re.match(r"\s*(\d+)\.(\d+)(?:\.(\d+))?", version)
if match is None:
return None
return tuple(int(part) for part in match.groups() if part is not None)
# Enable HIP online tuning early, before hipBLASLt initializes.
# Turn on hipBLASLt online tuning if use AITER hipBLASLt GEMM.
if (
+4
View File
@@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]):
return tokenizer
def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
"""Render OpenAI completion suffix input when the renderer supports FIM."""
return None
def _decode(self, *args, **kwargs):
return self.get_tokenizer().decode(*args, **kwargs)
+7
View File
@@ -16,6 +16,10 @@ from .inputs import DictPrompt
from .inputs.preprocess import parse_dec_only_prompt
from .params import ChatParams
_FIM_BEGIN = "<|fim▁begin|>"
_FIM_HOLE = "<|fim▁hole|>"
_FIM_END = "<|fim▁end|>"
class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
def __init__(
@@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
def _apply_chat_template(self, *args, **kwargs):
return self.get_tokenizer().apply_chat_template(*args, **kwargs)
def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}"
def render_messages(
self,
messages: list[ChatCompletionMessageParam],
+53 -4
View File
@@ -4,7 +4,7 @@ import time
from collections.abc import Sequence
from dataclasses import dataclass
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, TypeAlias
from typing import TYPE_CHECKING, Any, TypeAlias, cast
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
from openai.types.responses.tool import Mcp, Tool
@@ -548,9 +548,58 @@ class OnlineRenderer:
Called directly by render_completion_request and delegated to by
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
"""
# Return error for unsupported features.
prompt_input = request.prompt
if request.suffix is not None:
return self.create_error_response("suffix is not currently supported")
if request.echo:
return self.create_error_response(
"Echo is unsupported with suffix.",
param="suffix",
)
if request.prompt_embeds is not None:
return self.create_error_response(
"suffix is not supported with prompt_embeds",
param="suffix",
)
if request.truncate_prompt_tokens is not None:
return self.create_error_response(
"suffix is not supported with truncate_prompt_tokens",
param="suffix",
)
if isinstance(request.prompt, str):
rendered_prompt = self.renderer.render_completion_suffix(
request.prompt, request.suffix
)
if rendered_prompt is None:
return self.create_error_response(
"suffix is only supported for models with FIM completion "
"rendering",
param="suffix",
)
prompt_input = rendered_prompt
elif isinstance(request.prompt, list) and all(
isinstance(prompt, str) for prompt in request.prompt
):
rendered_prompts = []
for prompt in cast(list[str], request.prompt):
rendered_prompt = self.renderer.render_completion_suffix(
prompt, request.suffix
)
if rendered_prompt is None:
return self.create_error_response(
"suffix is only supported for models with FIM completion "
"rendering",
param="suffix",
)
rendered_prompts.append(rendered_prompt)
prompt_input = rendered_prompts
else:
return self.create_error_response(
"suffix requires text prompt input for FIM completion rendering",
param="suffix",
)
if request.echo and request.prompt_embeds is not None:
return self.create_error_response("Echo is unsupported with prompt embeds.")
@@ -562,7 +611,7 @@ class OnlineRenderer:
engine_inputs = await self.preprocess_completion(
request,
prompt_input=request.prompt,
prompt_input=prompt_input,
prompt_embeds=request.prompt_embeds,
skip_mm_cache=skip_mm_cache,
)
+5
View File
@@ -316,8 +316,13 @@ class ScalarType:
class scalar_types:
uint2 = ScalarType.uint(2, None)
uint3 = ScalarType.uint(3, None)
int4 = ScalarType.int_(4, None)
uint4 = ScalarType.uint(4, None)
uint5 = ScalarType.uint(5, None)
uint6 = ScalarType.uint(6, None)
uint7 = ScalarType.uint(7, None)
int8 = ScalarType.int_(8, None)
uint8 = ScalarType.uint(8, None)
float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN)
+1 -1
View File
@@ -110,7 +110,7 @@ class OvisProcessor(ProcessorMixin):
) -> BatchFeature:
"""Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwargs` arguments to
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`.
Args:
@@ -115,7 +115,7 @@ class Ovis2_5Processor(ProcessorMixin):
and image(s). This method forwards the `text`and `kwargs` arguments
to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text`
is not `None` to encode the text. To prepare the vision inputs,
this method forwards the `vision_infos` and `kwrags` arguments to
this method forwards the `vision_infos` and `kwargs` arguments to
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`]
if `vision_infos` is not `None`.
+24
View File
@@ -926,6 +926,10 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
"""
return False
def fused_qk_norm_mrope_kvcache_supported(self):
"""Whether this implementation supports fused QKNorm+MRoPE+KVCache."""
return False
def fused_rope_kvcache_supported(self):
"""Does this attention implementation support RoPE+KVCache fusion.
This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops
@@ -955,6 +959,26 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
"""
raise NotImplementedError
def do_qk_norm_mrope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
mrope_section: tuple[int, int, int],
is_interleaved: bool,
rotary_dim: int,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
"""Apply QK-norm and MRoPE, then write K/V to the cache."""
raise NotImplementedError
def do_rope_and_kv_cache_update(
self,
layer: AttentionLayer,
+9
View File
@@ -406,6 +406,15 @@ class CPUAttentionBackendImpl(AttentionImpl):
kv_cache_dtype=self.kv_cache_dtype,
)
# The CPU kernel executes attention sinks natively in bf16. If the
# sinks tensor is anything other than bf16, cast it to fp32 so it is
# executed in full float precision (done lazily here, after weights
# are loaded, rather than at __init__ time).
if self.sinks is not None and self.sinks.dtype not in [
torch.bfloat16,
torch.float32,
]:
self.sinks = self.sinks.to(torch.float32)
ops.cpu_attention_with_kv_cache(
query=query[:num_actual_tokens],
key_cache=key_cache,
@@ -772,9 +772,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device=device,
)
from vllm.platforms import current_platform
from vllm.v1.worker.workspace import current_workspace_manager
max_num_partial_tiles = reduce_partial_map_size
# AITER metadata sizing assumes all requests can carry max_prefill_qlen tokens,
# which is a loose worst case in the number of QO tiles. Rather, the sum of
# qlens is bounded by max_num_batched_tokens. Manually compute the number of
# qo tiles to avoid OOM at startup.
# TODO: AITER should give us this budget constrained value
qo_tile_cnt = (
cdiv(max_num_batched_tokens, _FP8_PREFILL_TILE_Q) + max_num_reqs - 1
)
max_num_partial_tiles = qo_tile_cnt + current_platform.num_compute_units()
reservations: list[tuple[tuple[int, ...], torch.dtype]] = [
(
(max_num_partial_tiles * _FP8_PREFILL_TILE_Q, num_head_k, v_head_dim),
@@ -7,7 +7,7 @@ from typing import ClassVar
import torch
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
from vllm.config import get_current_vllm_config_or_none
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
@@ -359,6 +359,9 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
def fused_qk_norm_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def fused_qk_norm_mrope_kvcache_supported(self) -> bool:
return IS_AITER_FOUND
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
@@ -397,6 +400,47 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
use_shuffle_layout=False,
)
def do_qk_norm_mrope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
mrope_section: tuple[int, int, int],
is_interleaved: bool,
rotary_dim: int,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
key_cache, value_cache = self._split_kv_cache(kv_cache)
rocm_aiter_ops.do_qk_norm_mrope_kvcache_update(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=self.num_heads,
num_heads_k=self.num_kv_heads,
head_dim=self.head_size,
is_neox=is_neox,
mrope_section=list(mrope_section),
is_interleaved=is_interleaved,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
key_cache=key_cache,
value_cache=value_cache,
slot_mapping=layer_slot_mapping,
k_scale=layer._k_scale_cpu,
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
rotary_dim=rotary_dim,
)
def do_rope_and_kv_cache_update(
self,
layer: AttentionLayer,
+1 -1
View File
@@ -437,7 +437,7 @@ class KVCacheManager:
----------------------------------------------------------------------
```
Abbrivations:
Abbreviations:
```
comp = request.num_computed_tokens
+26
View File
@@ -555,6 +555,11 @@ def _post_update_kernel(
all_token_ids_ptr,
all_token_ids_stride,
total_len_ptr,
broadcast_drafts_ptr,
broadcast_drafts_stride,
draft_tokens_ptr,
draft_tokens_stride,
num_spec,
):
req_id = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + req_id)
@@ -562,6 +567,17 @@ def _post_update_kernel(
# Filter rows with negative index entries.
return
if broadcast_drafts_ptr is not None:
# PP path: adopt the draft tokens proposed by the last rank's
# speculator so the next verification step embeds the real drafts.
for i in range(num_spec):
token_id = tl.load(
broadcast_drafts_ptr + req_id * broadcast_drafts_stride + i
)
tl.store(
draft_tokens_ptr + req_state_idx * draft_tokens_stride + i, token_id
)
total_len = tl.load(total_len_ptr + req_state_idx)
num_sampled = tl.load(num_sampled_ptr + req_id)
if num_sampled > 0:
@@ -622,6 +638,11 @@ def post_update(
all_token_ids: torch.Tensor,
# [max_num_reqs]
total_len: torch.Tensor,
# [num_reqs, num_spec]; drafts broadcast from the last PP rank. Only
# passed on non-last PP ranks, which never run the speculator.
broadcast_drafts: torch.Tensor | None = None,
# [max_num_reqs, num_spec]
draft_tokens_out: torch.Tensor | None = None,
) -> None:
num_reqs = idx_mapping.shape[0]
_post_update_kernel[(num_reqs,)](
@@ -638,6 +659,11 @@ def post_update(
all_token_ids,
all_token_ids.stride(0),
total_len,
broadcast_drafts,
broadcast_drafts.stride(0) if broadcast_drafts is not None else 0,
draft_tokens_out,
draft_tokens_out.stride(0) if draft_tokens_out is not None else 0,
broadcast_drafts.shape[1] if broadcast_drafts is not None else 0,
num_warps=1,
)
+63 -5
View File
@@ -860,6 +860,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# dummy run the eagle speculator's propose to ensure DP/EP sync.
if self.speculator is not None:
assert self.sampler is not None
assert hidden_states is not None
self.step_timing.drafter_start()
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
if self.speculator.supports_mm_inputs:
@@ -875,7 +876,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec_hidden_states = hidden_states
if hasattr(self.model, "get_mtp_target_hidden_states"):
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
if pre_hc_hidden_states is not None:
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
@@ -1107,6 +1109,44 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if outputs is not None:
self.postprocess_sampled(**outputs)
def warmup_pp_decode_update(self) -> None:
"""JIT-compile the kernel behind ``update_pp_decode_requests``.
That path only runs on real steps, so the warmup steps never reach it
on non-last PP ranks. Its first triton compile must not happen
mid-serving: the in-flight sampled-token broadcast keeps a NCCL kernel
spinning on this device, which blocks the CUDA module load and
deadlocks the pipeline. An all -1 idx_mapping makes this a no-op.
The freshly allocated int32 tensors are 16-byte aligned, matching the
padded views `PPHandler` produces at serving time (triton specializes
on pointer alignment).
"""
assert self.pp_handler is not None
num_spec = self.pp_handler.max_sample_len - 1
broadcast_drafts = (
torch.zeros((1, num_spec), dtype=torch.int64, device=self.device)
if num_spec > 0
else None
)
post_update(
torch.full((1,), -1, dtype=torch.int64, device=self.device),
self.req_states.num_computed_tokens.gpu,
self.req_states.last_sampled_tokens,
None,
torch.zeros(
(1, self.pp_handler.max_sample_len),
dtype=torch.int64,
device=self.device,
),
torch.zeros(1, dtype=torch.int32, device=self.device),
torch.zeros(1, dtype=torch.int32, device=self.device),
None,
self.req_states.all_token_ids.gpu,
self.req_states.total_len.gpu,
broadcast_drafts,
self.req_states.draft_tokens if broadcast_drafts is not None else None,
)
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
for new_req_data in scheduler_output.scheduled_new_reqs:
assert new_req_data.prefill_token_ids is not None
@@ -1588,6 +1628,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
query_start_loc: torch.Tensor | None = None,
broadcast_drafts: torch.Tensor | None = None,
) -> None:
# Update the number of computed tokens.
output_bin_counts = None
@@ -1605,6 +1646,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
query_start_loc,
self.req_states.all_token_ids.gpu,
self.req_states.total_len.gpu,
broadcast_drafts,
self.req_states.draft_tokens if broadcast_drafts is not None else None,
)
self.model_state.postprocess_state(
@@ -2125,7 +2168,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec_hidden_states = draft_hidden_states
if hasattr(self.model, "get_mtp_target_hidden_states"):
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
if pre_hc_hidden_states is not None:
spec_hidden_states = pre_hc_hidden_states[
: draft_hidden_states.size(0)
]
if isinstance(self.sampler, GPUWatermarkSampler):
self.speculator.prepare_watermarking(
self.sampler._get_contexts(input_batch.idx_mapping),
@@ -2147,8 +2193,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dp_sync=dp_sync,
mm_inputs=mm_inputs,
)
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
if self.adaptive_verification is not None:
if draft_tokens is not None:
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
if self.pp_handler is not None:
# Earlier stages never run the speculator; ship the
# drafts so their next verification step embeds the real
# draft tokens instead of stale buffer contents.
self.pp_handler.broadcast_drafts(
self.req_states.draft_tokens, input_batch
)
if draft_tokens is not None and self.adaptive_verification is not None:
self.adaptive_verification.record_confidences(
self.speculator.draft_token_confidence_probs, input_batch
)
@@ -2160,7 +2214,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch,
self.req_states.draft_tokens[input_batch.idx_mapping],
)
if self.pp_handler is not None:
if self.pp_handler is not None and self.speculator is None:
# When a speculator ran, the propose() path above already
# broadcast the fresh drafts. Broadcasting here as well would
# double-post on the pp_broadcast group and misalign the
# recv FIFO on earlier stages, hanging the pipeline.
self.pp_handler.broadcast_drafts(
self.req_states.draft_tokens, input_batch
)
+65 -20
View File
@@ -46,6 +46,18 @@ def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
return produces_sample if produces_sample.any() else None
def _alloc_combined(num_reqs: int, device: torch.device) -> torch.Tensor:
"""Allocate the (2, N) int32 buffer broadcast alongside sampled tokens.
The inner dim is padded to a multiple of 4 so that both `unbind` views
stay 16-byte aligned for any `num_reqs`: triton specializes on pointer
alignment, and a misaligned `num_rejected` would JIT-compile a second
`_post_update_kernel` variant at serving time. The padding is broadcast
but never read. Sender and receiver must both use this allocation.
"""
return torch.empty(2, -(-num_reqs // 4) * 4, dtype=torch.int32, device=device)
class PPHandler:
"""Runs the PP sampled-token broadcast/recv on a side stream so the
default stream isn't gated by the matching peer call. Step T's recv is
@@ -86,6 +98,14 @@ class PPHandler:
)
self.aux_hidden_state_relay_keys: tuple[str, ...] = ()
# Warmup steps run the pipeline with synthetic batches whose outputs are
# discarded; the sampled-token broadcast is disabled there so its
# side-stream NCCL ops cannot overlap the next step's activation p2p.
self.disabled = False
def set_disabled(self, disabled: bool) -> None:
self.disabled = disabled
def on_req_idx_freed(self, req_idx: int) -> None:
self.req_idx_gen_np[req_idx] += 1
@@ -157,29 +177,15 @@ class PPHandler:
num_sampled=slot.num_sampled,
num_rejected=slot.num_rejected,
idx_mapping=idx_mapping,
broadcast_drafts=slot.draft_tokens,
)
def broadcast_drafts(
self, draft_tokens: torch.Tensor, input_batch: InputBatch
) -> None:
"""Broadcast draft proposals so non-last ranks can embed real token ids."""
assert self.is_last_rank
if compute_need_sampled_mask(input_batch) is None:
return
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
send = draft_tokens[input_batch.idx_mapping].contiguous()
# Must record the idx_mapping tensor since it was allocated
# on the main stream.
input_batch.idx_mapping.record_stream(self.broadcast_stream)
torch.distributed.broadcast(
send, src=self.last_rank, group=self.broadcast_group
)
def receive(self, input_batch: InputBatch) -> bool:
"""Returns True iff sampled tokens need to be gathered from *all*
requests in the batch."""
assert not self.is_last_rank
if self.disabled:
return False
need_sampled_mask = compute_need_sampled_mask(input_batch)
if need_sampled_mask is None:
# Leave this step's reserved slot as None.
@@ -195,7 +201,7 @@ class PPHandler:
sampled_tokens = torch.empty(
num_reqs, self.max_sample_len, dtype=torch.int64, device=self.device
)
combined = torch.empty(2, num_reqs, dtype=torch.int32, device=self.device)
combined = _alloc_combined(num_reqs, self.device)
torch.distributed.broadcast(
sampled_tokens, src=self.last_rank, group=self.broadcast_group
)
@@ -213,6 +219,7 @@ class PPHandler:
torch.distributed.broadcast(
draft_tokens, src=self.last_rank, group=self.broadcast_group
)
event = self.broadcast_stream.record_event()
num_sampled, num_rejected = combined.unbind(dim=0)
# Must record_stream since these were allocated on broadcast stream but
@@ -242,11 +249,16 @@ class PPHandler:
input_batch: InputBatch,
) -> None:
assert self.is_last_rank
if compute_need_sampled_mask(input_batch) is None:
if self.disabled:
return
mask = compute_need_sampled_mask(input_batch)
if mask is None:
# No request needs sampled outputs for a subsequent decode step.
return
assert sampled_token_ids.dtype == torch.int64
assert num_sampled.dtype == torch.int32
assert num_rejected.dtype == torch.int32
if current_platform.is_xpu():
self.main_stream.synchronize()
@@ -262,9 +274,42 @@ class PPHandler:
src=self.last_rank,
group=self.broadcast_group,
)
combined = torch.stack((num_sampled, num_rejected), dim=0)
combined = _alloc_combined(num_sampled.shape[0], self.device)
combined[0, : num_sampled.shape[0]] = num_sampled
combined[1, : num_sampled.shape[0]] = num_rejected
torch.distributed.broadcast(
combined, src=self.last_rank, group=self.broadcast_group
)
for tensor in (sampled_token_ids, num_sampled, num_rejected):
tensor.record_stream(self.broadcast_stream)
def broadcast_drafts(
self, draft_token_table: torch.Tensor, input_batch: InputBatch
) -> None:
"""Broadcast the speculator's freshly proposed draft tokens.
Runs after propose() on the last rank; the send is stream-ordered
after broadcast()'s sends, matching receive()'s enqueue order. The
payload is gathered from the runner's draft table (just updated from
propose()'s output) into a fresh compact tensor: the speculator
overwrites its own persistent buffer on the next step, possibly
before this async send completes.
"""
assert self.is_last_rank
if self.disabled or self.max_sample_len == 1:
return
if compute_need_sampled_mask(input_batch) is None:
return
assert draft_token_table.dtype == torch.int64
assert draft_token_table.shape[1] == self.max_sample_len - 1
# Gather on the main stream so the payload is ordered after this
# step's table update and before any later one.
drafts = draft_token_table[input_batch.idx_mapping]
assert drafts.shape[0] == input_batch.num_reqs
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
torch.distributed.broadcast(
drafts, src=self.last_rank, group=self.broadcast_group
)
drafts.record_stream(self.broadcast_stream)
@@ -256,6 +256,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
is_padding=self.input_buffers.is_padding[:num_tokens],
):
last_hidden_states = self.model(
input_ids=self.input_buffers.input_ids[:num_tokens],
@@ -487,6 +488,7 @@ def _prepare_dflash_inputs_kernel(
# Outputs
out_input_ids_ptr,
out_query_positions_ptr,
out_is_padding_ptr,
out_query_start_loc_ptr,
out_seq_lens_ptr,
out_query_slot_mapping_ptr,
@@ -617,6 +619,7 @@ def _prepare_dflash_inputs_kernel(
)
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
tl.store(out_is_padding_ptr + query_idx, False, mask=is_query)
clamped_query_pos = tl.minimum(query_pos, max_model_len - 1)
tl.store(out_query_positions_ptr + query_idx, clamped_query_pos, mask=is_query)
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)
@@ -679,6 +682,9 @@ def _prepare_dflash_inputs_kernel(
for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE):
block = i + tl.arange(0, BLOCK_SIZE)
mask = block < max_num_tokens
tl.store(out_input_ids_ptr + block, 0, mask=mask)
tl.store(out_query_positions_ptr + block, 0, mask=mask)
tl.store(out_is_padding_ptr + block, True, mask=mask)
tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask)
@@ -730,6 +736,7 @@ def prepare_dflash_inputs(
_prepare_dflash_inputs_kernel[(num_reqs, num_blocks)](
input_buffers.input_ids,
input_buffers.positions,
input_buffers.is_padding,
input_buffers.query_start_loc,
input_buffers.seq_lens,
query_slot_mapping,
+5
View File
@@ -445,6 +445,11 @@ def _warmup_kernels(
for step_indices, step_spec_flags in decode_steps:
_run_decode_step(step_indices, step_spec_flags)
# The deferred PP post-update path only runs on real steps, so the steps
# above never JIT-compile its kernel on non-last ranks.
if not model_runner.is_last_pp_rank and model_runner.pp_handler is not None:
model_runner.warmup_pp_decode_update()
# Clean up - process finish_req_ids.
cleanup_output = SchedulerOutput.make_empty()
cleanup_output.finished_req_ids = set(req_ids)
+12
View File
@@ -805,6 +805,15 @@ class Worker(WorkerBase):
@instrument(span_name="Warmup (GPU)")
def compile_or_warm_up_model(self) -> CompilationTimes:
# All warmup phases below run synthetic steps whose sampled outputs are
# discarded. The PP sampled-token broadcast would carry no payload, and
# its side-stream NCCL ops can overlap the next step's activation p2p
# and deadlock the pipeline, so keep it disabled for the whole warmup
# window and restore it before serving.
pp_handler = getattr(self.model_runner, "pp_handler", None)
if pp_handler is not None:
pp_handler.set_disabled(True)
warmup_sizes: list[int] = []
if (
@@ -983,6 +992,9 @@ class Worker(WorkerBase):
# gate so subsequent `execute_model` / `sample_tokens` calls enforce it.
enable_gpu_sync_check()
if pp_handler is not None:
pp_handler.set_disabled(False)
return CompilationTimes(
language_model=self.compilation_config.compilation_time,
encoder=self.compilation_config.encoder_compilation_time,