Merge branch 'main' into qwen4exp-qsa-fused-draft-metadata
This commit is contained in:
@@ -1818,6 +1818,48 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py
|
||||
|
||||
- label: ":amd: (MI355) Kimi K3" # TBD
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_1
|
||||
num_gpus: 1
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/models/kimi_k3/
|
||||
- csrc/libtorch_stable/kimi_k3/
|
||||
- tests/models/kimi_k3/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
# Mirrors the ":amd: (MI355) Kimi K3" step generated from
|
||||
# .buildkite/test_areas/models_basic.yaml. The ROCm KDA/AttnRes kernels are
|
||||
# gated on gfx950, so this has to be MI355. The two kernels/ files the NVIDIA
|
||||
# step runs are CUDA-only (they import cuda.bindings.driver), so they are
|
||||
# deliberately left out here.
|
||||
- pytest -v -s models/kimi_k3
|
||||
|
||||
- label: ":amd: (MI355) Kimi K3 Multi-GPU" # TBD
|
||||
timeout_in_minutes: 30
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_4
|
||||
num_gpus: 4
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/models/kimi_k3/
|
||||
- tests/models/kimi_k3/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
# The single-GPU step above skips every @multi_gpu_test case. `-m "distributed"`
|
||||
# selects exactly those: multi_gpu_test() applies pytest.mark.distributed
|
||||
# alongside its skipif (tests/utils.py:2080), so this picks up a new
|
||||
# multi-GPU test anywhere in the directory without editing this list.
|
||||
# Four GPUs run the two tp4 cases in test_amd_latent_moe_runner.py; the tp8
|
||||
# case and the 8/16-GPU cases in test_latent_moe_tail.py still skip.
|
||||
- pytest -v -s -m "distributed" models/kimi_k3
|
||||
|
||||
- label: ":amd: (MI300) Mamba Kernels" # TBD
|
||||
timeout_in_minutes: 55
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||
@@ -3562,6 +3604,49 @@ steps:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_4
|
||||
num_gpus: 4
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
|
||||
- tests/utils.py
|
||||
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
|
||||
- vllm/platforms/rocm.py
|
||||
- requirements/common.txt
|
||||
- requirements/kv_connectors_rocm.txt
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
|
||||
|
||||
- label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
|
||||
timeout_in_minutes: 30
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_4
|
||||
num_gpus: 4
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
|
||||
- tests/v1/kv_connector/mooncake_integration/
|
||||
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
|
||||
- tests/utils.py
|
||||
- examples/disaggregated/mooncake_connector/
|
||||
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
|
||||
- vllm/platforms/rocm.py
|
||||
- requirements/common.txt
|
||||
- requirements/kv_connectors_rocm.txt
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
|
||||
|
||||
- label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" # TBD
|
||||
timeout_in_minutes: 40
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
|
||||
@@ -42,6 +42,29 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
num_devices: 4
|
||||
timeout_in_minutes: 60
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
|
||||
- tests/utils.py
|
||||
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
|
||||
- vllm/platforms/rocm.py
|
||||
- requirements/common.txt
|
||||
- requirements/kv_connectors_rocm.txt
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
# Preserve all six model/TP pairs with native AITER attention.
|
||||
- pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
|
||||
|
||||
- label: ":nvidia: (L4) Push NixlConnector PP prefill PD accuracy"
|
||||
key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
|
||||
|
||||
@@ -19,6 +19,30 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
|
||||
dind: false
|
||||
device: mi355_4
|
||||
num_devices: 4
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
|
||||
- tests/v1/kv_connector/mooncake_integration/
|
||||
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
|
||||
- tests/utils.py
|
||||
- examples/disaggregated/mooncake_connector/
|
||||
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
|
||||
- vllm/platforms/rocm.py
|
||||
- requirements/common.txt
|
||||
- requirements/kv_connectors_rocm.txt
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
# Same full accuracy/TP sweep, using same-host HIP GPU transport.
|
||||
- pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
|
||||
|
||||
- label: ":nvidia: (L4) Mooncake EC TCP E2E"
|
||||
key: mooncake-ec-tcp-e2e-2-gpus
|
||||
|
||||
@@ -12,6 +12,7 @@ steps:
|
||||
working_dir: "."
|
||||
env:
|
||||
SNAPSHOT_E2E_RUN_LIMIT_S: "1200"
|
||||
NVIDIA_VISIBLE_DEVICES: "0"
|
||||
source_file_dependencies:
|
||||
- .buildkite/scripts/initialized-snapshot-e2e.sh
|
||||
- .buildkite/test_areas/entrypoints.yaml
|
||||
|
||||
@@ -571,7 +571,7 @@ steps:
|
||||
- label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY"
|
||||
key: lm-eval-humming-h100-act
|
||||
optional: true
|
||||
timeout_in_minutes: 70
|
||||
timeout_in_minutes: 90
|
||||
device: h100
|
||||
num_devices: 1
|
||||
source_file_dependencies:
|
||||
@@ -582,7 +582,7 @@ steps:
|
||||
- vllm/model_executor/kernels/linear/
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
|
||||
# - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
|
||||
|
||||
- label: ":nvidia: (H100) LM Eval Humming DeepEP TEMPORARY"
|
||||
key: lm-eval-humming-h100-deepep
|
||||
|
||||
@@ -109,6 +109,23 @@ steps:
|
||||
commands:
|
||||
# The native NVIDIA Kimi K3 kernels require the SM100 family.
|
||||
- pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Kimi K3"
|
||||
dind: false
|
||||
device: mi355_1
|
||||
num_devices: 1
|
||||
timeout_in_minutes: 60
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
# The ROCm KDA/AttnRes kernels are gated on gfx950, so this has to be MI355.
|
||||
# The two kernels/ files the NVIDIA job adds are CUDA-only (they import
|
||||
# cuda.bindings.driver), so they are deliberately left out here.
|
||||
- pytest -v -s models/kimi_k3
|
||||
|
||||
- label: ":nvidia: (H200) GLM5Next"
|
||||
key: glm5next-unit-tests
|
||||
|
||||
+13
-1
@@ -243,7 +243,19 @@ void cpu_attention_with_kv_cache(
|
||||
input.block_table = block_table.data_ptr<int32_t>();
|
||||
input.alibi_slopes =
|
||||
alibi_slopes.has_value() ? alibi_slopes->data_ptr<float>() : nullptr;
|
||||
input.s_aux = s_aux.has_value() ? s_aux->data_ptr<c10::BFloat16>() : nullptr;
|
||||
// Attention sinks may be bf16 (native path) or fp32. Anything that is not
|
||||
// bf16 must be provided as fp32 and is executed in full float precision.
|
||||
if (s_aux.has_value()) {
|
||||
TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 ||
|
||||
s_aux->scalar_type() == at::ScalarType::Float,
|
||||
"cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ",
|
||||
s_aux->scalar_type(), " must be bfloat16 or float32");
|
||||
input.s_aux = s_aux->data_ptr();
|
||||
input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16;
|
||||
} else {
|
||||
input.s_aux = nullptr;
|
||||
input.s_aux_is_bf16 = false;
|
||||
}
|
||||
input.dynamic_causal =
|
||||
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
|
||||
input.scale = scale;
|
||||
|
||||
@@ -879,7 +879,11 @@ struct AttentionInput {
|
||||
int32_t* seq_lens;
|
||||
int32_t* block_table;
|
||||
float* alibi_slopes;
|
||||
c10::BFloat16* s_aux;
|
||||
// Attention sinks pointer. May reference bf16 or fp32 data depending on
|
||||
// `s_aux_is_bf16`. bf16 sinks keep the native bf16 path; anything else is
|
||||
// provided as fp32 and executed in full float precision.
|
||||
const void* s_aux;
|
||||
bool s_aux_is_bf16;
|
||||
bool* dynamic_causal;
|
||||
float scale;
|
||||
bool causal;
|
||||
@@ -1565,7 +1569,8 @@ class AttentionMainLoop {
|
||||
const float scale = input->scale;
|
||||
const float softcap_scale = input->softcap;
|
||||
const float* alibi_slopes = input->alibi_slopes;
|
||||
const c10::BFloat16* s_aux = input->s_aux;
|
||||
const void* s_aux = input->s_aux;
|
||||
const bool s_aux_is_bf16 = input->s_aux_is_bf16;
|
||||
const bool* dynamic_causal = input->dynamic_causal;
|
||||
const bool is_dynamic_causal = dynamic_causal != nullptr;
|
||||
|
||||
@@ -1773,9 +1778,6 @@ class AttentionMainLoop {
|
||||
const float* curr_alibi_slopes =
|
||||
(alibi_slopes != nullptr ? alibi_slopes + q_head_start_idx
|
||||
: nullptr);
|
||||
const c10::BFloat16* curr_s_aux =
|
||||
(s_aux != nullptr ? s_aux + q_head_start_idx : nullptr);
|
||||
|
||||
// copy the Q tile to q_buffer, the logical layout of q_buffer is
|
||||
// [actual_q_token_num, curr_q_heads_per_kv, head_dim]
|
||||
{
|
||||
@@ -1787,10 +1789,23 @@ class AttentionMainLoop {
|
||||
|
||||
if (use_sink) {
|
||||
alignas(64) float s_aux_fp32[16];
|
||||
// All other platforms have BF16Vec16 available
|
||||
vec_op::BF16Vec16 vec_bf16(curr_s_aux);
|
||||
vec_op::FP32Vec16 vec_fp32(vec_bf16);
|
||||
vec_fp32.save(s_aux_fp32);
|
||||
// Sinks may be provided as bf16 or fp32. Either way they are
|
||||
// loaded into fp32: bf16 sinks are up-converted (native
|
||||
// BF16Vec16 path, available on all platforms), while fp32
|
||||
// sinks are loaded directly and executed in full precision.
|
||||
if (s_aux_is_bf16) {
|
||||
const c10::BFloat16* curr_s_aux =
|
||||
static_cast<const c10::BFloat16*>(s_aux) +
|
||||
q_head_start_idx;
|
||||
vec_op::BF16Vec16 vec_bf16(curr_s_aux);
|
||||
vec_op::FP32Vec16 vec_fp32(vec_bf16);
|
||||
vec_fp32.save(s_aux_fp32);
|
||||
} else {
|
||||
const float* curr_s_aux =
|
||||
static_cast<const float*>(s_aux) + q_head_start_idx;
|
||||
vec_op::FP32Vec16 vec_fp32(curr_s_aux);
|
||||
vec_fp32.save(s_aux_fp32);
|
||||
}
|
||||
|
||||
float* __restrict__ curr_sum_buffer = sum_buffer;
|
||||
float* __restrict__ curr_max_buffer = max_buffer;
|
||||
|
||||
@@ -145,6 +145,9 @@ When batch invariance is enabled, vLLM:
|
||||
1. Uses deterministic kernel implementations for attention and other operations
|
||||
2. Ensures consistent numerical behavior across different batch sizes
|
||||
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode)
|
||||
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
|
||||
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
|
||||
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
|
||||
|
||||
!!! note
|
||||
Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility.
|
||||
|
||||
@@ -549,6 +549,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
|
||||
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
|
||||
| `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | ✅︎ | ✅︎ |
|
||||
| `DiffusionGemmaForBlockDiffusion` | DiffusionGemma | T + I<sup>+</sup> + V<sup>+</sup> | `google/diffusiongemma-26B-A4B-it`, etc. | | |
|
||||
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
|
||||
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
|
||||
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -33,4 +33,4 @@ quack-kernels==0.6.5 # Required by MSA
|
||||
tokenspeed-mla==0.1.8; platform_system == "Linux"
|
||||
|
||||
# Humming kernels for quantization gemm
|
||||
humming-kernels[cu13]==0.1.15
|
||||
humming-kernels[cu13]==0.1.16
|
||||
|
||||
@@ -128,8 +128,11 @@ FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
|
||||
),
|
||||
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
# The combined pass can consume these sites before the standalone pass.
|
||||
"norm_rope_fusion": re.compile(
|
||||
r"qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on (\d+) sites"
|
||||
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
|
||||
r"qk_norm_rope_kvcache_fusion.py:\d+] "
|
||||
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
|
||||
),
|
||||
"attn_quant_fusion": re.compile(
|
||||
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
|
||||
|
||||
@@ -182,7 +182,11 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
|
||||
# TODO: Remove log counting in unit tests
|
||||
# once all matchers implement VllmFusionPatternMatcherPass
|
||||
n_expected = tp_size * num_ranges_activated
|
||||
if match_name not in ("attn_quant_fusion", "act_quant_fusion"):
|
||||
if match_name not in (
|
||||
"attn_quant_fusion",
|
||||
"act_quant_fusion",
|
||||
"norm_rope_fusion",
|
||||
):
|
||||
assert len(log_matches) == n_expected, (
|
||||
f"Could not find {n_expected} {match_name} "
|
||||
f"(found {len(log_matches)}) in:\n {log_holder.text}"
|
||||
@@ -190,7 +194,16 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
|
||||
|
||||
expected_matches = getattr(matches, match_name)
|
||||
|
||||
if match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check:
|
||||
if match_name == "norm_rope_fusion":
|
||||
# Opaque LayerName lets the combined pass consume short-range
|
||||
# sites before the standalone pass; count either owner.
|
||||
assert len(log_matches) >= n_expected
|
||||
assert all(m in (0, expected_matches) for m in log_matches)
|
||||
assert sum(log_matches) == expected_matches * n_expected, (
|
||||
f"{match_name} expected {expected_matches * n_expected} "
|
||||
f"sites across both passes, found: {log_matches}"
|
||||
)
|
||||
elif match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check:
|
||||
# AR+rms+quant takes precedence over rms+quant if activated.
|
||||
# That means we get full matching where ar+rms+quant was not
|
||||
# activated, and less where it was (only the smallest range).
|
||||
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
|
||||
import vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion as fusion_module
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.v1.attention.utils import (
|
||||
@@ -15,8 +16,12 @@ from tests.v1.attention.utils import (
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
|
||||
from vllm._aiter_ops import (
|
||||
IS_AITER_FOUND,
|
||||
is_aiter_found_and_supported,
|
||||
rocm_aiter_ops,
|
||||
)
|
||||
from vllm.compilation.passes.fusion.matcher_utils import MROPE_OP, ROTARY_OP
|
||||
from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import (
|
||||
QkNormRopeKvCacheFusionPass,
|
||||
)
|
||||
@@ -37,8 +42,12 @@ from vllm.config import (
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
@@ -94,6 +103,7 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
device: torch.device,
|
||||
rotary_dim: int | None = None,
|
||||
prefix: str = "model.layers.0.self_attn.attn",
|
||||
expected_query_quant_group_shape: GroupShape | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
@@ -133,6 +143,9 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
attn_backend=attn_backend.get_class(),
|
||||
)
|
||||
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
|
||||
if expected_query_quant_group_shape is not None:
|
||||
assert self.attn.query_quant is not None
|
||||
assert self.attn.query_quant.group_shape == expected_query_quant_group_shape
|
||||
assert not self.attn_backend.forward_includes_kv_cache_update, (
|
||||
f"Attention backend {self.attn_backend} does not support "
|
||||
"fuse_qk_norm_rope_kvcache."
|
||||
@@ -227,18 +240,15 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
self.kv_cache_dtype != self.dtype
|
||||
and self.attn.impl.supports_quant_query_input
|
||||
):
|
||||
q_fp8 = torch.empty_like(q, dtype=FP8_DTYPE)
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant(
|
||||
q_fp8, q, self.attn._q_scale, False
|
||||
)
|
||||
q = q_fp8
|
||||
assert self.attn.query_quant is not None
|
||||
q, _ = self.attn.query_quant(q, self.attn._q_scale)
|
||||
|
||||
# Final views + KV cache update
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size)
|
||||
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
|
||||
k, v, self.layer_name
|
||||
k, v, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
return q, k, v, kv_cache_dummy_dep
|
||||
|
||||
@@ -260,6 +270,64 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default]
|
||||
|
||||
|
||||
class QKNormMRoPEKVCacheTestModel(QKNormRoPEKVCacheTestModel):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
mrope_section: tuple[int, int, int],
|
||||
mrope_interleaved: bool,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.rotary_emb = MRotaryEmbedding(
|
||||
self.head_size,
|
||||
rotary_dim=self.rotary_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=self.is_neox,
|
||||
dtype=self.dtype,
|
||||
mrope_section=list(mrope_section),
|
||||
mrope_interleaved=mrope_interleaved,
|
||||
)
|
||||
self.enable_rope_custom_op = self.rotary_emb.enabled()
|
||||
|
||||
def forward(
|
||||
self, qkv: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q, k, v, kv_cache_dummy_dep = super().forward(qkv, positions)
|
||||
output = torch.empty(
|
||||
q.shape[0],
|
||||
self.num_heads * self.head_size,
|
||||
device=q.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
torch.ops.vllm.unified_attention_with_output(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output,
|
||||
_encode_layer_name(self.layer_name),
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
# Production decoder attention consumes K from the cache rather than
|
||||
# its direct K argument. Returning `output` keeps that production user
|
||||
# in the graph without exposing K to an unsupported consumer.
|
||||
return q, output, v, kv_cache_dummy_dep
|
||||
|
||||
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
|
||||
return [MROPE_OP, torch.ops.vllm.unified_kv_cache_update.default]
|
||||
|
||||
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
|
||||
return [torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default]
|
||||
|
||||
|
||||
class QKNormMRoPEObservableKTestModel(QKNormMRoPEKVCacheTestModel):
|
||||
def forward(
|
||||
self, qkv: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return QKNormRoPEKVCacheTestModel.forward(self, qkv, positions)
|
||||
|
||||
|
||||
def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
*,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
@@ -278,18 +346,33 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mrope_section: tuple[int, int, int] | None = None,
|
||||
mrope_interleaved: bool = False,
|
||||
observe_mrope_k: bool = False,
|
||||
expect_fusion: bool = True,
|
||||
use_quark_scalar_query_scale: bool = False,
|
||||
) -> None:
|
||||
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Set AITER toggles before VllmConfig initialization enables env caching.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
cache_dtype=kv_cache_dtype,
|
||||
),
|
||||
quant_config=(
|
||||
QuarkConfig(quant_config={"exclude": []})
|
||||
if use_quark_scalar_query_scale
|
||||
else None
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=[custom_op],
|
||||
@@ -302,6 +385,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
m.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
|
||||
m.setenv(
|
||||
"VLLM_ROCM_USE_AITER_TRITON_ROPE",
|
||||
"1" if enable_aiter_triton_rope else "0",
|
||||
@@ -309,18 +393,49 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
model = QKNormRoPEKVCacheTestModel(
|
||||
vllm_config=vllm_config,
|
||||
attn_backend=attn_backend,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
rotary_dim=rotary_dim,
|
||||
is_neox=is_neox,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
dtype=dtype,
|
||||
device=torch.get_default_device(),
|
||||
)
|
||||
expected_query_quant_group_shape = None
|
||||
if use_quark_scalar_query_scale:
|
||||
expected_query_quant_group_shape = (
|
||||
GroupShape(-1, num_heads * head_size)
|
||||
if num_kv_heads == 1
|
||||
else GroupShape.PER_TENSOR
|
||||
)
|
||||
|
||||
model_kwargs = {
|
||||
"vllm_config": vllm_config,
|
||||
"attn_backend": attn_backend,
|
||||
"num_heads": num_heads,
|
||||
"num_kv_heads": num_kv_heads,
|
||||
"head_size": head_size,
|
||||
"rotary_dim": rotary_dim,
|
||||
"is_neox": is_neox,
|
||||
"rms_norm_eps": rms_norm_eps,
|
||||
"dtype": dtype,
|
||||
"device": torch.get_default_device(),
|
||||
"expected_query_quant_group_shape": expected_query_quant_group_shape,
|
||||
}
|
||||
if mrope_section is None:
|
||||
model = QKNormRoPEKVCacheTestModel(**model_kwargs)
|
||||
else:
|
||||
alternate_section = (8, 8, 8)
|
||||
m.setattr(
|
||||
fusion_module,
|
||||
"_discover_mrope_configs",
|
||||
lambda _: (
|
||||
(mrope_section, mrope_interleaved),
|
||||
(alternate_section, not mrope_interleaved),
|
||||
),
|
||||
)
|
||||
model_cls = (
|
||||
QKNormMRoPEObservableKTestModel
|
||||
if observe_mrope_k
|
||||
else QKNormMRoPEKVCacheTestModel
|
||||
)
|
||||
model = model_cls(
|
||||
**model_kwargs,
|
||||
mrope_section=mrope_section,
|
||||
mrope_interleaved=mrope_interleaved,
|
||||
)
|
||||
|
||||
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
|
||||
passes = [
|
||||
@@ -337,7 +452,12 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
num_heads * head_size + 2 * num_kv_heads * head_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
pos = torch.arange(num_tokens, dtype=torch.long)
|
||||
if mrope_section is None:
|
||||
pos = torch.arange(num_tokens, dtype=torch.long)
|
||||
else:
|
||||
# Real Qwen-VL position IDs can be a strided [3, N] view.
|
||||
pos = torch.randint(0, 4096, (3, 2 * num_tokens), dtype=torch.long)[:, ::2]
|
||||
assert not pos.is_contiguous()
|
||||
|
||||
qkv_unfused = qkv.clone()
|
||||
pos_unfused = pos.clone()
|
||||
@@ -356,7 +476,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
|
||||
# Run fused (compiled) forward
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0 if mrope_section is None else 1)
|
||||
with set_forward_context(None, vllm_config):
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
forward_context = get_forward_context()
|
||||
@@ -369,14 +489,33 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
kv_cache_fused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
assert fusion_pass.matched_count == int(expect_fusion)
|
||||
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
if expect_fusion:
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
else:
|
||||
for op in model.ops_in_model_before():
|
||||
assert backend.op_count(op, before=True) > 0
|
||||
assert backend.op_count(op) > 0
|
||||
|
||||
if use_quark_scalar_query_scale:
|
||||
expected_quant_op = (
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant.default
|
||||
if num_kv_heads != 1
|
||||
else torch.ops._C.static_scaled_fp8_quant.default
|
||||
)
|
||||
assert backend.op_count(expected_quant_op, before=True) > 0
|
||||
# Query quantization remains separate after the prologue fusion.
|
||||
assert backend.op_count(expected_quant_op) > 0
|
||||
|
||||
# Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2;
|
||||
# AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2.
|
||||
ATOL, RTOL = (2e-2, 2e-2) if enable_aiter_triton_rope else (1e-2, 1e-2)
|
||||
ATOL, RTOL = (
|
||||
(2e-2, 2e-2)
|
||||
if enable_aiter_triton_rope or mrope_section is not None
|
||||
else (1e-2, 1e-2)
|
||||
)
|
||||
is_fp8_cache = model.kv_cache_dtype != dtype
|
||||
|
||||
if q_fused.dtype == FP8_DTYPE:
|
||||
@@ -387,7 +526,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
else:
|
||||
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
if not is_fp8_cache:
|
||||
if not expect_fusion or (not is_fp8_cache and mrope_section is None):
|
||||
# The AITER PTS kernel populates k_out only for non-FP8 caches.
|
||||
# With FP8, the kernel writes quantized K directly to the cache
|
||||
# and may leave k_out uninitialised. In production this is fine
|
||||
@@ -397,15 +536,16 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
# Should be bit exact since no processing had been done on v for both paths
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
|
||||
|
||||
# fp8 vs triton-rope ref requires loosening tolerance to 1.25e-1.
|
||||
if is_fp8_cache and enable_aiter_triton_rope:
|
||||
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
|
||||
# Allow one E4M3 quantization step while still comparing every block.
|
||||
if is_fp8_cache:
|
||||
cache_atol = cache_rtol = 1.25e-1
|
||||
else:
|
||||
cache_atol, cache_rtol = ATOL, RTOL
|
||||
|
||||
torch.testing.assert_close(
|
||||
kv_cache_unfused[0].float(),
|
||||
kv_cache_fused[0].float(),
|
||||
kv_cache_unfused.float(),
|
||||
kv_cache_fused.float(),
|
||||
atol=cache_atol,
|
||||
rtol=cache_rtol,
|
||||
)
|
||||
@@ -494,3 +634,225 @@ def test_qk_norm_rope_kvcache_fusion(
|
||||
custom_op=custom_op,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mrope_section",
|
||||
[
|
||||
pytest.param((16, 24, 24), id="full_rotary"),
|
||||
pytest.param((4, 6, 6), id="partial_rotary"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("mrope_interleaved", [False, True])
|
||||
@pytest.mark.parametrize("is_neox", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_layout",
|
||||
[
|
||||
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
|
||||
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
|
||||
pytest.param(KVCacheLayout.LHBNC, id="block_major"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.skipif(
|
||||
not IS_AITER_FOUND,
|
||||
reason="Requires AITER",
|
||||
)
|
||||
def test_qk_norm_mrope_kvcache_fusion(
|
||||
mrope_section: tuple[int, int, int],
|
||||
mrope_interleaved: bool,
|
||||
is_neox: bool,
|
||||
kv_layout: KVCacheLayout,
|
||||
kv_cache_dtype: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
enable_aiter_triton_rope=False,
|
||||
num_tokens=5,
|
||||
num_heads=8,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
rotary_dim=2 * sum(mrope_section),
|
||||
block_size=16,
|
||||
is_neox=is_neox,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_layout=kv_layout,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
rms_norm_eps=1e-6,
|
||||
custom_op="+rotary_embedding",
|
||||
monkeypatch=monkeypatch,
|
||||
mrope_section=mrope_section,
|
||||
mrope_interleaved=mrope_interleaved,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_heads", "num_kv_heads"),
|
||||
[(32, 2), (16, 1), (8, 1)],
|
||||
ids=["tp2", "tp4", "tp8"],
|
||||
)
|
||||
def test_qk_norm_mrope_kvcache_fusion_quark_scalar_query_scale(
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
enable_aiter_triton_rope=False,
|
||||
num_tokens=5,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=128,
|
||||
rotary_dim=128,
|
||||
block_size=16,
|
||||
is_neox=True,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_layout=KVCacheLayout.LBHNC,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
rms_norm_eps=1e-6,
|
||||
custom_op="+rotary_embedding",
|
||||
monkeypatch=monkeypatch,
|
||||
mrope_section=(24, 20, 20),
|
||||
mrope_interleaved=True,
|
||||
use_quark_scalar_query_scale=True,
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_fusion_rejects_observable_k(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
enable_aiter_triton_rope=False,
|
||||
num_tokens=5,
|
||||
num_heads=8,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
rotary_dim=128,
|
||||
block_size=16,
|
||||
is_neox=True,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_layout=KVCacheLayout.LBHNC,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
rms_norm_eps=1e-6,
|
||||
custom_op="+rotary_embedding",
|
||||
monkeypatch=monkeypatch,
|
||||
mrope_section=(16, 24, 24),
|
||||
observe_mrope_k=True,
|
||||
expect_fusion=False,
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_fusion_rejects_unaligned_rotary_dim(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
enable_aiter_triton_rope=False,
|
||||
num_tokens=5,
|
||||
num_heads=8,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
rotary_dim=6,
|
||||
block_size=16,
|
||||
is_neox=True,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_layout=KVCacheLayout.LBHNC,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
rms_norm_eps=1e-6,
|
||||
custom_op="+rotary_embedding",
|
||||
monkeypatch=monkeypatch,
|
||||
mrope_section=(1, 1, 1),
|
||||
expect_fusion=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _USE_LAYERNAME, reason="requires opaque LayerName")
|
||||
def test_opaque_layer_name_patterns_register_once(monkeypatch: pytest.MonkeyPatch):
|
||||
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=torch.bfloat16),
|
||||
cache_config=CacheConfig(block_size=16, cache_dtype="fp8_e4m3"),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rotary_embedding"],
|
||||
pass_config=PassConfig(fuse_qk_norm_rope_kvcache=True),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
model_kwargs = {
|
||||
"vllm_config": vllm_config,
|
||||
"attn_backend": AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
"num_heads": 8,
|
||||
"num_kv_heads": 2,
|
||||
"head_size": 128,
|
||||
"is_neox": True,
|
||||
"rms_norm_eps": 1e-6,
|
||||
"dtype": torch.bfloat16,
|
||||
"device": torch.device(device),
|
||||
}
|
||||
models = [
|
||||
QKNormRoPEKVCacheTestModel(
|
||||
**model_kwargs,
|
||||
prefix=f"model.layers.{index}.self_attn.attn",
|
||||
)
|
||||
for index in range(2)
|
||||
]
|
||||
|
||||
# Opaque layer names are wildcards in the pattern graph. Registering
|
||||
# once per physical layer raises "Duplicate pattern" on torch >= 2.11.
|
||||
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
|
||||
|
||||
assert models
|
||||
assert fusion_pass.patterns
|
||||
|
||||
|
||||
def _call_fused_mrope_impl(q_out: torch.Tensor) -> None:
|
||||
device = q_out.device
|
||||
fusion_module.fused_qk_norm_mrope_and_unified_kv_cache_update_impl(
|
||||
q_out=q_out,
|
||||
qkv=torch.zeros(2, 16, device=device),
|
||||
positions=torch.arange(2, device=device),
|
||||
q_weight=torch.ones(4, device=device),
|
||||
k_weight=torch.ones(4, device=device),
|
||||
rms_norm_eps=1e-6,
|
||||
cos_sin_cache=torch.zeros(16, 4, device=device),
|
||||
is_neox=True,
|
||||
mrope_section_t=1,
|
||||
mrope_section_h=1,
|
||||
mrope_section_w=0,
|
||||
is_interleaved=False,
|
||||
rotary_dim=4,
|
||||
layer_name="model.layers.0.self_attn.attn",
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_profiling_pass_initializes_query(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(
|
||||
fusion_module,
|
||||
"get_attention_context",
|
||||
lambda _: (None, None, None, None),
|
||||
)
|
||||
q_out = torch.full((2, 8), torch.nan, device="cpu")
|
||||
|
||||
_call_fused_mrope_impl(q_out)
|
||||
|
||||
torch.testing.assert_close(q_out, torch.zeros_like(q_out))
|
||||
|
||||
|
||||
def test_mrope_attention_context_failure_propagates(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
def fail_context(_):
|
||||
raise RuntimeError("missing attention context")
|
||||
|
||||
monkeypatch.setattr(fusion_module, "get_attention_context", fail_context)
|
||||
|
||||
with pytest.raises(RuntimeError, match="missing attention context"):
|
||||
_call_fused_mrope_impl(torch.empty(2, 8, device="cpu"))
|
||||
|
||||
@@ -14,8 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
@@ -24,6 +26,9 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
MODEL_NAME_SHORT = "gpt2"
|
||||
DEEPSEEK_V4_FIM_BEGIN = "<|fim▁begin|>"
|
||||
DEEPSEEK_V4_FIM_HOLE = "<|fim▁hole|>"
|
||||
DEEPSEEK_V4_FIM_END = "<|fim▁end|>"
|
||||
_PER_REQUEST_STATS = RequestStateStats(
|
||||
queued_ts=1.0,
|
||||
scheduled_ts=1.5,
|
||||
@@ -302,6 +307,179 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
|
||||
assert response.metrics.speculative_decoding.num_spec_steps == 2
|
||||
|
||||
|
||||
def _build_online_renderer_for_completion(
|
||||
model_config: MockModelConfig,
|
||||
renderer: Any | None = None,
|
||||
) -> OnlineRenderer:
|
||||
if renderer is None:
|
||||
renderer = MagicMock()
|
||||
renderer.render_completion_suffix.return_value = None
|
||||
|
||||
return OnlineRenderer(
|
||||
model_config=model_config,
|
||||
renderer=renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
|
||||
def _deepseek_v4_suffix_renderer(
|
||||
model_config: MockModelConfig,
|
||||
) -> DeepseekV4Renderer:
|
||||
return DeepseekV4Renderer(
|
||||
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_completion_suffix_uses_renderer_capability():
|
||||
model_config = MockModelConfig()
|
||||
online_renderer = _build_online_renderer_for_completion(
|
||||
model_config,
|
||||
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||
)
|
||||
online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="def fib(n):\n return ",
|
||||
suffix="\n\nprint(fib(10))",
|
||||
max_tokens=64,
|
||||
)
|
||||
|
||||
result = await online_renderer.render_completion(request)
|
||||
|
||||
assert result == [{"ok": True}]
|
||||
online_renderer.preprocess_completion.assert_awaited_once()
|
||||
assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == (
|
||||
f"{DEEPSEEK_V4_FIM_BEGIN}def fib(n):\n return "
|
||||
f"{DEEPSEEK_V4_FIM_HOLE}\n\nprint(fib(10))"
|
||||
f"{DEEPSEEK_V4_FIM_END}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_completion_suffix_supports_text_prompt_list():
|
||||
model_config = MockModelConfig()
|
||||
online_renderer = _build_online_renderer_for_completion(
|
||||
model_config,
|
||||
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||
)
|
||||
online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt=["prefix A", "prefix B"],
|
||||
suffix=" suffix",
|
||||
)
|
||||
|
||||
result = await online_renderer.render_completion(request)
|
||||
|
||||
assert result == [{"ok": True}]
|
||||
assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [
|
||||
f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
|
||||
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
|
||||
f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
|
||||
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_completion_suffix_rejects_renderer_without_fim_support():
|
||||
model_config = MockModelConfig()
|
||||
online_renderer = _build_online_renderer_for_completion(model_config)
|
||||
online_renderer.preprocess_completion = AsyncMock()
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="prefix",
|
||||
suffix="suffix",
|
||||
)
|
||||
|
||||
result = await online_renderer.render_completion(request)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert "FIM completion rendering" in result.error.message
|
||||
online_renderer.preprocess_completion.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_completion_suffix_rejects_echo():
|
||||
model_config = MockModelConfig()
|
||||
online_renderer = _build_online_renderer_for_completion(
|
||||
model_config,
|
||||
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||
)
|
||||
online_renderer.preprocess_completion = AsyncMock()
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="prefix",
|
||||
suffix="suffix",
|
||||
echo=True,
|
||||
)
|
||||
|
||||
result = await online_renderer.render_completion(request)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert "Echo is unsupported with suffix" in result.error.message
|
||||
online_renderer.preprocess_completion.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_completion_suffix_rejects_prompt_embeds():
|
||||
model_config = MockModelConfig()
|
||||
online_renderer = _build_online_renderer_for_completion(
|
||||
model_config,
|
||||
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||
)
|
||||
online_renderer.preprocess_completion = AsyncMock()
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="prefix",
|
||||
suffix="suffix",
|
||||
prompt_embeds=b"embeds",
|
||||
)
|
||||
|
||||
result = await online_renderer.render_completion(request)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert "suffix is not supported with prompt_embeds" in result.error.message
|
||||
online_renderer.preprocess_completion.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("prompt", ([1, 2, 3], [[1, 2, 3]]))
|
||||
async def test_completion_suffix_rejects_token_prompt(prompt):
|
||||
model_config = MockModelConfig()
|
||||
online_renderer = _build_online_renderer_for_completion(
|
||||
model_config,
|
||||
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||
)
|
||||
online_renderer.preprocess_completion = AsyncMock()
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt=prompt,
|
||||
suffix="suffix",
|
||||
)
|
||||
|
||||
result = await online_renderer.render_completion(request)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert "requires text prompt input" in result.error.message
|
||||
online_renderer.preprocess_completion.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_error_non_stream():
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
|
||||
|
||||
@@ -3,6 +3,5 @@ accuracy_threshold: 0.39
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--linear-backend humming
|
||||
|
||||
@@ -3,7 +3,6 @@ accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
|
||||
accuracy_threshold: 0.78
|
||||
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-a4"
|
||||
accuracy_threshold: 0.70
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-asym"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--linear-backend humming
|
||||
@@ -1,3 +1,4 @@
|
||||
gpt-oss-20b-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
|
||||
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
|
||||
Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
|
||||
|
||||
@@ -4,4 +4,4 @@ Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
|
||||
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
|
||||
Qwen3-4B-mixed-quant-RTN-humming.yaml
|
||||
Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
|
||||
|
||||
@@ -428,6 +428,7 @@ def varlen_with_paged_kv(
|
||||
k_scale: float = 1.0,
|
||||
v_scale: float = 1.0,
|
||||
dynamic_causal: list[bool] | None = None,
|
||||
s_aux_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> None:
|
||||
set_random_seed(0)
|
||||
num_seqs = len(seq_lens)
|
||||
@@ -449,9 +450,7 @@ def varlen_with_paged_kv(
|
||||
# 2^(-8/n)
|
||||
alibi_slopes = _get_alibi_slopes(num_query_heads) if use_alibi else None
|
||||
|
||||
s_aux = (
|
||||
15 * torch.rand((num_query_heads,), dtype=torch.bfloat16) if use_sink else None
|
||||
)
|
||||
s_aux = 15 * torch.rand((num_query_heads,), dtype=s_aux_dtype) if use_sink else None
|
||||
|
||||
is_fp8 = kv_cache_dtype != "auto"
|
||||
if is_fp8 and current_platform.get_cpu_architecture() != CpuArchEnum.X86:
|
||||
@@ -538,6 +537,8 @@ def varlen_with_paged_kv(
|
||||
)
|
||||
|
||||
out_without_split = torch.empty_like(query)
|
||||
if s_aux is not None and s_aux.dtype != torch.bfloat16:
|
||||
s_aux = s_aux.to(torch.float32)
|
||||
cpu_attention_with_kv_cache(
|
||||
query=query,
|
||||
key_cache=packed_key_cache,
|
||||
@@ -1128,6 +1129,51 @@ def test_varlen_with_paged_kv_sink(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3"])
|
||||
@pytest.mark.parametrize("seq_lens", SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", [96])
|
||||
@pytest.mark.parametrize("block_size", [128])
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("soft_cap", [None])
|
||||
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
|
||||
@pytest.mark.parametrize("use_alibi", [False])
|
||||
@pytest.mark.parametrize("use_sink", [True])
|
||||
@pytest.mark.parametrize("isa", [get_attn_isa()])
|
||||
@pytest.mark.parametrize("s_aux_dtype", [torch.float16])
|
||||
def test_varlen_with_paged_kv_sink_fp16(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
sliding_window: int | None,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
num_blocks: int,
|
||||
use_alibi: bool,
|
||||
use_sink: bool,
|
||||
isa: str,
|
||||
kv_cache_dtype: str,
|
||||
s_aux_dtype: torch.dtype,
|
||||
) -> None:
|
||||
varlen_with_paged_kv(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
sliding_window=sliding_window,
|
||||
dtype=dtype,
|
||||
block_size=block_size,
|
||||
soft_cap=soft_cap,
|
||||
num_blocks=num_blocks,
|
||||
use_alibi=use_alibi,
|
||||
use_sink=use_sink,
|
||||
isa=isa,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
s_aux_dtype=s_aux_dtype,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
[
|
||||
|
||||
@@ -107,7 +107,16 @@ SEEDS = [0]
|
||||
|
||||
|
||||
def torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
w1_s,
|
||||
w2_s,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
block_shape,
|
||||
silu_fp32=False,
|
||||
expert_map=None,
|
||||
):
|
||||
"""Fused MoE with block-wise fp8 quantization using native torch.
|
||||
|
||||
@@ -123,6 +132,9 @@ def torch_w8a8_block_fp8_moe(
|
||||
|
||||
topk_weight = topk_weight.view(-1)
|
||||
topk_ids = topk_ids.view(-1)
|
||||
if expert_map is not None:
|
||||
# w1/w2 hold this rank's experts; slots mapped to -1 contribute nothing.
|
||||
topk_ids = expert_map[topk_ids]
|
||||
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
# Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the
|
||||
@@ -301,6 +313,54 @@ def test_w8a8_block_fp8_clamp_skips_fused_silu_quant(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), [(83, 512, 512), (2048, 4608, 512)])
|
||||
@pytest.mark.parametrize("topk", [2, 6])
|
||||
@pytest.mark.parametrize("ep_size", [2, 4])
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_fused_moe_expert_map(M, N, K, topk, ep_size, workspace_init):
|
||||
"""One EP rank: local expert shard plus an expert_map with -1 entries."""
|
||||
E, block_size, dtype = 16, [128, 128], torch.bfloat16
|
||||
torch.manual_seed(0)
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
w1, w2, quant_config = make_test_quant_config(
|
||||
E, N, K, dtype, quant_dtype=torch.float8_e4m3fn, block_shape=block_size
|
||||
)
|
||||
|
||||
local_e = E // ep_size
|
||||
e_ids = torch.randperm(E, dtype=torch.int32)[:local_e]
|
||||
e_map = torch.full((E,), -1, dtype=torch.int32)
|
||||
e_map[e_ids] = torch.arange(local_e, dtype=torch.int32)
|
||||
w1, w2 = w1[e_ids].contiguous(), w2[e_ids].contiguous()
|
||||
w1_s = quant_config.w1_scale[e_ids].contiguous()
|
||||
w2_s = quant_config.w2_scale[e_ids].contiguous()
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s, w2_scale=w2_s, block_shape=block_size
|
||||
)
|
||||
|
||||
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size, expert_map=e_map
|
||||
)
|
||||
m_out = m_fused_moe.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=e_map,
|
||||
global_num_experts=E,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(m_out, ref_out, atol=0.035, rtol=0.035)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
|
||||
@pytest.mark.parametrize("E", E)
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
|
||||
@@ -339,6 +339,136 @@ def test_dsv4_fast_topk(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="The DeepSeek V4 fast path is CUDA-only.",
|
||||
)
|
||||
def test_dsv4_fast_topk_padding_uint32_falls_back(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Padded rows need the -1 sentinel, which uint32 cannot represent: the
|
||||
router must skip the dsv4 fast path and still route the real rows."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 17
|
||||
num_experts = 256
|
||||
hidden_states = torch.randn((num_tokens, 64), dtype=torch.float32, device="cuda")
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
is_padding = torch.zeros(num_tokens, dtype=torch.bool, device="cuda")
|
||||
is_padding[1::2] = True
|
||||
gating_output[is_padding] = float("nan")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.router."
|
||||
"fused_topk_bias_router._get_padding_mask",
|
||||
lambda _: is_padding,
|
||||
)
|
||||
# uint32 + padding would trip dsv4_topk's signed-indices assertion; the
|
||||
# generic path must take over instead.
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=correction_bias,
|
||||
topk=6,
|
||||
renormalize=True,
|
||||
indices_type=torch.uint32,
|
||||
routed_scaling_factor=1.5,
|
||||
)
|
||||
|
||||
assert topk_ids.dtype == torch.uint32
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output[~is_padding],
|
||||
topk=6,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=correction_bias,
|
||||
)
|
||||
# uint32 CUDA tensors do not support boolean-mask indexing; widen first.
|
||||
torch.testing.assert_close(
|
||||
topk_ids.to(torch.int64)[~is_padding],
|
||||
topk_ids_ref.to(torch.int64),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
topk_weights[~is_padding], topk_weights_ref, atol=2e-5, rtol=2e-5
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="The DeepSeek V4 fast path is CUDA-only.",
|
||||
)
|
||||
def test_dsv4_fast_topk_padding(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Verify the DSV4 fast path removes graph-padding rows from routing."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 17
|
||||
num_experts = 256
|
||||
hidden_states = torch.randn((num_tokens, 64), dtype=torch.float32, device="cuda")
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
is_padding = torch.zeros(num_tokens, dtype=torch.bool, device="cuda")
|
||||
is_padding[1::2] = True
|
||||
gating_output[is_padding] = float("nan")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.router."
|
||||
"fused_topk_bias_router._get_padding_mask",
|
||||
lambda _: is_padding,
|
||||
)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=correction_bias,
|
||||
topk=6,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.5,
|
||||
)
|
||||
|
||||
assert torch.equal(topk_ids[is_padding], torch.full_like(topk_ids[is_padding], -1))
|
||||
assert torch.equal(
|
||||
topk_weights[is_padding], torch.zeros_like(topk_weights[is_padding])
|
||||
)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output[~is_padding],
|
||||
topk=6,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=correction_bias,
|
||||
)
|
||||
torch.testing.assert_close(topk_ids[~is_padding], topk_ids_ref, atol=0, rtol=0)
|
||||
torch.testing.assert_close(
|
||||
topk_weights[~is_padding], topk_weights_ref, atol=2e-5, rtol=2e-5
|
||||
)
|
||||
|
||||
# The mask buffer is persistent under CUDA graph replay, but its contents
|
||||
# change with every batch. Verify that the kernel reads those contents at
|
||||
# runtime rather than specializing on the mask captured above.
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_weights, graph_ids = dsv4_topk(
|
||||
gating_output,
|
||||
correction_bias,
|
||||
torch.int32,
|
||||
1.5,
|
||||
is_padding=is_padding,
|
||||
)
|
||||
|
||||
is_padding.logical_not_()
|
||||
graph.replay()
|
||||
assert torch.equal(
|
||||
graph_ids[is_padding], torch.full_like(graph_ids[is_padding], -1)
|
||||
)
|
||||
assert torch.equal(
|
||||
graph_weights[is_padding], torch.zeros_like(graph_weights[is_padding])
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
|
||||
@@ -17,6 +17,26 @@ EPS = 1e-5
|
||||
attn_res_module = importlib.import_module("vllm.models.kimi_k3.nvidia.ops.attn_res")
|
||||
|
||||
|
||||
def _on_rocm_below_10() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import get_rocm_version
|
||||
|
||||
return (get_rocm_version() or (0,)) < (10,)
|
||||
|
||||
|
||||
# The Triton bundled with ROCm < 10 (3.7.x) crashes in the AMD
|
||||
# CanonicalizePointers pass on the kernel's tl.where over pointer tensors. Only
|
||||
# the num_blocks > 0 loop contains it. ROCm serves AttnRes from
|
||||
# vllm/models/kimi_k3/amd/ops/attn_res.py instead, so nothing real is lost.
|
||||
_OLD_ROCM = _on_rocm_below_10()
|
||||
|
||||
|
||||
def _skip_on_old_rocm(num_blocks: int) -> None:
|
||||
if _OLD_ROCM and num_blocks > 0:
|
||||
pytest.skip("Triton on ROCm < 10 cannot compile this kernel's pointer select")
|
||||
|
||||
|
||||
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
|
||||
storage = torch.randn(
|
||||
*shape[:-1],
|
||||
@@ -79,6 +99,7 @@ def test_attn_res(
|
||||
):
|
||||
if backend == "nvidia" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("NVIDIA AttnRes requires the SM100 family")
|
||||
_skip_on_old_rocm(num_blocks)
|
||||
|
||||
prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
|
||||
delta = (
|
||||
@@ -133,6 +154,7 @@ def test_attn_res(
|
||||
|
||||
@pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1))
|
||||
def test_attn_res_block_counts(num_blocks: int):
|
||||
_skip_on_old_rocm(num_blocks)
|
||||
prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
blocks = torch.randn(
|
||||
1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
@@ -169,6 +191,7 @@ def test_attn_res_block_counts(num_blocks: int):
|
||||
|
||||
|
||||
def test_attn_res_without_output_norm():
|
||||
_skip_on_old_rocm(MAX_BLOCKS)
|
||||
prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
delta = torch.randn_like(prefix)
|
||||
blocks = torch.randn(
|
||||
|
||||
@@ -855,7 +855,7 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
dynamic_mxfp4_config = _make_qtensor_config(
|
||||
mxfp4_config = _make_qtensor_config(
|
||||
{
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
@@ -866,24 +866,23 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
|
||||
None,
|
||||
exclude=["self_attn.q_proj", "mlp.down_proj"],
|
||||
)
|
||||
dynamic_mxfp4_config.dynamic_mxfp4_quant = True
|
||||
|
||||
assert dynamic_mxfp4_config.get_quant_method_target(
|
||||
"self_attn.q_proj", LinearBase
|
||||
) == (kMxfp4Static, None, QuarkLinearMethod)
|
||||
attention_proj = TestLinear()
|
||||
assert isinstance(
|
||||
dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"),
|
||||
QuarkLinearMethod,
|
||||
assert mxfp4_config.get_quant_method_target("self_attn.q_proj", LinearBase) == (
|
||||
None,
|
||||
None,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
assert isinstance(attention_proj.scheme, QuarkOCP_MX)
|
||||
assert attention_proj.scheme.dynamic_mxfp4_quant
|
||||
|
||||
assert dynamic_mxfp4_config.get_quant_method_target(
|
||||
"mlp.down_proj", LinearBase
|
||||
) == (None, None, UnquantizedLinearMethod)
|
||||
assert isinstance(
|
||||
dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
|
||||
mxfp4_config.get_quant_method(TestLinear(), "self_attn.q_proj"),
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
assert mxfp4_config.get_quant_method_target("mlp.down_proj", LinearBase) == (
|
||||
None,
|
||||
None,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
|
||||
+66
-1
@@ -12,6 +12,7 @@ from unittest.mock import patch
|
||||
|
||||
import pydantic
|
||||
import pytest
|
||||
import torch
|
||||
from huggingface_hub import ResolvedRevision
|
||||
from pydantic import ValidationError
|
||||
|
||||
@@ -37,7 +38,7 @@ from vllm.config import (
|
||||
WatermarkConfig,
|
||||
update_config,
|
||||
)
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode, PassConfig
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
@@ -656,6 +657,70 @@ def test_breakable_cudagraph_platform_default(
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case,hidden,heads,intermediate,tp,expected",
|
||||
[
|
||||
("bf16", 2048, (16, 8), 6144, 1, True),
|
||||
("bi-off", 2048, (16, 8), 6144, 1, False),
|
||||
("opt-out", 2048, (16, 8), 6144, 1, False),
|
||||
("eager", 2048, (16, 8), 6144, 1, False),
|
||||
("sp", 2048, (16, 8), 6144, 1, False),
|
||||
("fp16", 2048, (16, 8), 6144, 1, False),
|
||||
("quantized", 2048, (16, 8), 6144, 1, False),
|
||||
("untuned", 4096, (32, 32), 11008, 1, False),
|
||||
("tp4-tuned", 4096, (8, 2), 12288, 4, True),
|
||||
("list-intermediate", 2048, (16, 8), [2048, 4096], 1, False),
|
||||
("no-table", 2048, None, 6144, 1, False),
|
||||
],
|
||||
)
|
||||
def test_batch_invariant_breakable_cudagraph(
|
||||
monkeypatch, case, hidden, heads, intermediate, tp, expected
|
||||
):
|
||||
from vllm.config.vllm import default_breakable_cudagraph_architectures
|
||||
from vllm.model_executor.determinism import batch_invariant_configs as bi_configs
|
||||
|
||||
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0" if case == "bi-off" else "1")
|
||||
monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False)
|
||||
if case == "opt-out":
|
||||
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0")
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
|
||||
table = None if case == "no-table" else {(12288, 2048): None, (6144, 4096): None}
|
||||
monkeypatch.setattr(current_platform, "get_device_capability", lambda: None)
|
||||
monkeypatch.setattr(bi_configs, "_get_tuned_matmul_arch_family", lambda cap: "test")
|
||||
monkeypatch.setattr(
|
||||
bi_configs,
|
||||
"_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS",
|
||||
{"test": table} if table else {},
|
||||
)
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
config = object.__new__(VllmConfig)
|
||||
config.model_config = SimpleNamespace(
|
||||
architectures=["Qwen3ForCausalLM"],
|
||||
enforce_eager=case == "eager",
|
||||
dtype=torch.float16 if case == "fp16" else torch.bfloat16,
|
||||
quantization="fp8" if case == "quantized" else None,
|
||||
hf_text_config=SimpleNamespace(intermediate_size=intermediate),
|
||||
get_hidden_size=lambda: hidden,
|
||||
get_head_size=lambda: 128,
|
||||
get_num_attention_heads=lambda pc: heads[0],
|
||||
get_num_kv_heads=lambda pc: heads[1],
|
||||
)
|
||||
config.parallel_config = SimpleNamespace(tensor_parallel_size=tp)
|
||||
config.compilation_config = (
|
||||
CompilationConfig(pass_config=PassConfig(enable_sp=True))
|
||||
if case == "sp"
|
||||
else CompilationConfig()
|
||||
)
|
||||
try:
|
||||
assert config._maybe_enable_breakable_cudagraph() is expected
|
||||
if expected:
|
||||
assert config.compilation_config.mode == CompilationMode.NONE
|
||||
finally:
|
||||
os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None)
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "expected_architecture"),
|
||||
[
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mooncake P/D accuracy with same-host HIP transport on ROCm."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import spawn_new_process_for_each_test
|
||||
from tests.v1.kv_connector.rocm_pd_accuracy_utils import (
|
||||
QWEN_MODEL,
|
||||
run_rocm_pd_accuracy,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"),
|
||||
pytest.mark.distributed(num_gpus=4),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,prefill_tp,decode_tp",
|
||||
[
|
||||
pytest.param(QWEN_MODEL, 1, 1, id="mooncake-hip-qwen-p1-d1"),
|
||||
pytest.param(QWEN_MODEL, 2, 2, id="mooncake-hip-qwen-p2-d2"),
|
||||
],
|
||||
)
|
||||
@spawn_new_process_for_each_test
|
||||
def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch):
|
||||
run_rocm_pd_accuracy("mooncake", model, prefill_tp, decode_tp, monkeypatch)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL P/D accuracy with AITER unified attention on ROCm."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import spawn_new_process_for_each_test
|
||||
from tests.v1.kv_connector.rocm_pd_accuracy_utils import (
|
||||
DEEPSEEK_MODEL,
|
||||
QWEN_MODEL,
|
||||
run_rocm_pd_accuracy,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"),
|
||||
pytest.mark.distributed(num_gpus=4),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,prefill_tp,decode_tp",
|
||||
[
|
||||
pytest.param(QWEN_MODEL, 2, 2, id="nixl-aiter-qwen-p2-d2"),
|
||||
pytest.param(QWEN_MODEL, 1, 2, id="nixl-aiter-qwen-p1-d2"),
|
||||
pytest.param(QWEN_MODEL, 2, 1, id="nixl-aiter-qwen-p2-d1"),
|
||||
pytest.param(DEEPSEEK_MODEL, 1, 1, id="nixl-aiter-deepseek-p1-d1"),
|
||||
pytest.param(DEEPSEEK_MODEL, 1, 2, id="nixl-aiter-deepseek-p1-d2"),
|
||||
pytest.param(DEEPSEEK_MODEL, 2, 1, id="nixl-aiter-deepseek-p2-d1"),
|
||||
],
|
||||
)
|
||||
@spawn_new_process_for_each_test
|
||||
def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch):
|
||||
run_rocm_pd_accuracy("nixl", model, prefill_tp, decode_tp, monkeypatch)
|
||||
@@ -0,0 +1,254 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared ROCm P/D accuracy harness for Mooncake and NIXL integration tests.
|
||||
|
||||
Keep the models, TP pairs, full GSM8K workload, and acceptance checks of the
|
||||
Mooncake and FlashInfer/NIXL sweeps. Mooncake uses same-host HIP IPC; NIXL uses
|
||||
AITER unified attention. These cases do not exercise RDMA or FlashInfer.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[3]
|
||||
QWEN_MODEL = "Qwen/Qwen3-0.6B"
|
||||
DEEPSEEK_MODEL = "deepseek-ai/deepseek-vl2-tiny"
|
||||
_REVISIONS = {
|
||||
QWEN_MODEL: "c1899de289a04d12100db370d81485cdf75e47ca",
|
||||
DEEPSEEK_MODEL: "66c54660eae7e90c9ba259bfdf92d07d6e3ce8aa",
|
||||
}
|
||||
|
||||
|
||||
def _metric_total(server, name: str, **labels: str) -> float:
|
||||
response = requests.get(server.url_for("metrics"), timeout=10)
|
||||
response.raise_for_status()
|
||||
return sum(
|
||||
sample.value
|
||||
for family in text_string_to_metric_families(response.text)
|
||||
for sample in family.samples
|
||||
if sample.name == name
|
||||
and all(sample.labels.get(key) == value for key, value in labels.items())
|
||||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _proxy(connector: str, prefill, decode, bootstrap_port: int):
|
||||
port = get_open_port()
|
||||
if connector == "mooncake":
|
||||
script = (
|
||||
_ROOT / "examples/disaggregated/mooncake_connector/"
|
||||
"mooncake_connector_proxy.py"
|
||||
)
|
||||
args = [
|
||||
"--prefill",
|
||||
prefill.url_root,
|
||||
str(bootstrap_port),
|
||||
"--decode",
|
||||
decode.url_root,
|
||||
]
|
||||
else:
|
||||
script = Path(__file__).parent / "nixl_integration/toy_proxy_server.py"
|
||||
args = [
|
||||
"--prefiller-hosts",
|
||||
"127.0.0.1",
|
||||
"--prefiller-ports",
|
||||
str(prefill.port),
|
||||
"--decoder-hosts",
|
||||
"127.0.0.1",
|
||||
"--decoder-ports",
|
||||
str(decode.port),
|
||||
]
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, str(script), "--port", str(port), *args],
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
assert process.poll() is None, "P/D proxy exited during startup"
|
||||
try:
|
||||
requests.get(f"http://127.0.0.1:{port}/", timeout=1)
|
||||
break
|
||||
except requests.ConnectionError:
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
raise TimeoutError("P/D proxy did not start")
|
||||
yield f"http://127.0.0.1:{port}/v1"
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=10)
|
||||
|
||||
|
||||
def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
|
||||
"""Transferred KV must feed decoding while full upstream accuracy holds."""
|
||||
# Preserve the caller's allocation instead of querying host-wide SMI indices.
|
||||
selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
|
||||
selector = next((key for key in selectors if key in os.environ), selectors[0])
|
||||
visible = os.getenv(selector)
|
||||
devices = (
|
||||
visible.split(",")
|
||||
if visible is not None
|
||||
else [str(i) for i in range(current_platform.device_count())]
|
||||
)
|
||||
assert len(devices) >= prefill_tp + decode_tp, "Insufficient allocated GPUs"
|
||||
for key in selectors:
|
||||
if key != selector:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
if connector == "mooncake":
|
||||
import mooncake.engine
|
||||
|
||||
assert mooncake.engine.SUPPORT_HIP, "Mooncake requires its ROCm/HIP build"
|
||||
connector_name = "MooncakeConnector"
|
||||
extra = {"mooncake_protocol": "hip", "device_name": ""}
|
||||
else:
|
||||
from vllm.distributed.nixl_utils import NixlWrapper
|
||||
|
||||
assert NixlWrapper is not None, "NIXL ROCm bindings must import"
|
||||
connector_name = "NixlConnector"
|
||||
extra = {}
|
||||
|
||||
common_args = [
|
||||
"--revision",
|
||||
_REVISIONS[model],
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--enforce-eager",
|
||||
"--distributed-executor-backend",
|
||||
"mp",
|
||||
"--block-size",
|
||||
"128",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--max-num-batched-tokens",
|
||||
"4096",
|
||||
"--gpu-memory-utilization",
|
||||
"0.2",
|
||||
"--kv-cache-memory-bytes",
|
||||
str(8 * 1024**3),
|
||||
"--moe-backend",
|
||||
"triton",
|
||||
"--attention-backend",
|
||||
"ROCM_AITER_UNIFIED_ATTN",
|
||||
]
|
||||
common_env = {
|
||||
"PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"],
|
||||
"VLLM_HOST_IP": "127.0.0.1",
|
||||
"VLLM_WORKER_MULTIPROC_METHOD": "spawn",
|
||||
"VLLM_USE_V2_MODEL_RUNNER": "1",
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_KV_CACHE_LAYOUT": "HND",
|
||||
}
|
||||
bootstrap_port = get_open_port()
|
||||
with contextlib.ExitStack() as stack:
|
||||
servers: list[RemoteOpenAIServer] = []
|
||||
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
|
||||
for role, tp, assigned in (
|
||||
("kv_producer", prefill_tp, devices[:prefill_tp]),
|
||||
("kv_consumer", decode_tp, devices[prefill_tp : prefill_tp + decode_tp]),
|
||||
):
|
||||
config = {
|
||||
"kv_connector": connector_name,
|
||||
"kv_role": role,
|
||||
"kv_connector_extra_config": extra,
|
||||
}
|
||||
env = {
|
||||
**common_env,
|
||||
selector: ",".join(assigned),
|
||||
"VLLM_PORT": str(get_open_port()),
|
||||
"VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()),
|
||||
"VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port),
|
||||
}
|
||||
servers.append(
|
||||
RemoteOpenAIServer(
|
||||
model,
|
||||
[
|
||||
*common_args,
|
||||
"--tensor-parallel-size",
|
||||
str(tp),
|
||||
"--kv-transfer-config",
|
||||
json.dumps(config),
|
||||
],
|
||||
env_dict=env,
|
||||
max_wait_seconds=1200,
|
||||
)
|
||||
)
|
||||
prefill, decode = servers
|
||||
proxy_url = stack.enter_context(
|
||||
_proxy(connector, prefill, decode, bootstrap_port)
|
||||
)
|
||||
|
||||
# Reuse the full upstream evaluation and its model-specific assertions.
|
||||
accuracy = importlib.import_module(
|
||||
f"tests.v1.kv_connector.{connector}_integration.test_accuracy"
|
||||
)
|
||||
monkeypatch.setattr(accuracy, "MODEL_NAME", model)
|
||||
monkeypatch.setattr(accuracy, "BASE_URL", proxy_url)
|
||||
assert model in accuracy.EXPECTED_VALUES
|
||||
|
||||
def transfer_totals():
|
||||
totals = {
|
||||
"external_tokens": _metric_total(
|
||||
decode,
|
||||
"vllm:prompt_tokens_by_source_total",
|
||||
source="external_kv_transfer",
|
||||
)
|
||||
}
|
||||
if connector == "nixl":
|
||||
totals["nixl_bytes"] = _metric_total(
|
||||
decode, "vllm:nixl_bytes_transferred_sum"
|
||||
)
|
||||
return totals
|
||||
|
||||
simple_evaluate = accuracy.lm_eval.simple_evaluate
|
||||
|
||||
def evaluate_with_transfer_checks(*args, **kwargs):
|
||||
# The upstream helper runs a smoke prompt before GSM8K. Wait until
|
||||
# its counters appear, then require new transfers during evaluation.
|
||||
deadline = time.monotonic() + 15
|
||||
before = transfer_totals()
|
||||
while not all(value > 0 for value in before.values()):
|
||||
assert time.monotonic() < deadline, (
|
||||
f"Smoke prompt did not transfer KV: {before}"
|
||||
)
|
||||
time.sleep(1)
|
||||
before = transfer_totals()
|
||||
|
||||
# The API adapter loads a local tokenizer because these proxies do
|
||||
# not expose /tokenize. Match it to the server's pinned checkpoint.
|
||||
kwargs["model_args"] += f",revision={_REVISIONS[model]}"
|
||||
results = simple_evaluate(*args, **kwargs)
|
||||
deadline = time.monotonic() + 15
|
||||
after = transfer_totals()
|
||||
while not all(after[key] > value for key, value in before.items()):
|
||||
assert time.monotonic() < deadline, (
|
||||
f"GSM8K did not transfer KV: before={before}, after={after}"
|
||||
)
|
||||
time.sleep(1)
|
||||
after = transfer_totals()
|
||||
print(f"GSM8K transfer counters: before={before}, after={after}")
|
||||
return results
|
||||
|
||||
monkeypatch.setattr(
|
||||
accuracy.lm_eval, "simple_evaluate", evaluate_with_transfer_checks
|
||||
)
|
||||
accuracy.test_accuracy()
|
||||
@@ -50,7 +50,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
ReadSpec,
|
||||
TPMapping,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
get_base_request_id,
|
||||
)
|
||||
@@ -344,6 +347,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
)
|
||||
|
||||
w._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list)
|
||||
w._send_failures = set()
|
||||
w._sending_transfers_lock = threading.Lock()
|
||||
w._push_finished_blocks = {}
|
||||
w._pending_d_registrations = {}
|
||||
@@ -941,9 +945,10 @@ class TestPushWriterNotifs:
|
||||
assert request_id not in w._sending_transfers
|
||||
|
||||
@pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)])
|
||||
def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma):
|
||||
"""P-side sends are completed by consumer notifs / lease expiry,
|
||||
not by local WRITE-handle completion."""
|
||||
def test_completed_push_send_releases_blocks_without_consumer_notif(
|
||||
self, pp_size, is_hma
|
||||
):
|
||||
"""All WRITEs completing lets P reclaim its blocks without a D ACK."""
|
||||
w = self._pollable_worker()
|
||||
w.pp_size = pp_size
|
||||
w._is_hma_required = is_hma
|
||||
@@ -951,20 +956,14 @@ class TestPushWriterNotifs:
|
||||
w.nixl_wrapper.check_xfer_state.side_effect = ["DONE", "PROC", "DONE"]
|
||||
|
||||
results = w.get_transfer_results()
|
||||
|
||||
assert request_id not in results.finished_sending
|
||||
assert request_id in w._reqs_to_send
|
||||
assert w._sending_transfers[request_id] == [102]
|
||||
assert request_id in w._reqs_to_send
|
||||
assert w._evict_finished_inbox.empty()
|
||||
|
||||
results = w.get_transfer_results()
|
||||
|
||||
assert request_id not in results.finished_sending
|
||||
assert request_id in w._reqs_to_send
|
||||
assert request_id not in w._sending_transfers
|
||||
|
||||
# The consumer notif is what completes the send.
|
||||
w._pending_completion_notifs.put(f"{request_id}:1".encode())
|
||||
results = w.get_transfer_results()
|
||||
assert request_id in results.finished_sending
|
||||
assert results.finished_sending == {request_id}
|
||||
assert request_id not in w._reqs_to_send
|
||||
assert request_id not in w._reqs_to_process
|
||||
assert w._evict_finished_inbox.get_nowait() == request_id
|
||||
@@ -973,6 +972,91 @@ class TestPushWriterNotifs:
|
||||
assert w.nixl_wrapper.release_xfer_handle.call_count == 2
|
||||
w.xfer_stats.record_kv_expired_req.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"])
|
||||
def test_push_failure_survives_until_lease_expiry(self, sibling_state):
|
||||
"""A later sibling completion must not turn a failed send into success."""
|
||||
w = self._pollable_worker()
|
||||
request_id = self._make_sending_req(w)
|
||||
w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state]
|
||||
|
||||
assert w.get_transfer_results().finished_sending == set()
|
||||
assert w._sending_transfers[request_id] == [102]
|
||||
assert w.get_transfer_results().finished_sending == set()
|
||||
assert request_id not in w._sending_transfers
|
||||
assert request_id in w._reqs_to_send
|
||||
assert w._evict_finished_inbox.empty()
|
||||
|
||||
w.expected_consumer_notifications_by_req = {}
|
||||
w._reqs_to_send[request_id] = time.perf_counter() - 1
|
||||
assert w.get_transfer_results().finished_sending == {request_id}
|
||||
assert request_id not in w._send_failures
|
||||
assert w._evict_finished_inbox.get_nowait() == request_id
|
||||
|
||||
@pytest.mark.parametrize("release_fails", [False, True])
|
||||
def test_push_submission_failure_is_remembered_after_sibling_success(
|
||||
self, release_fails
|
||||
):
|
||||
"""A submission error remains a failure even if later polls succeed."""
|
||||
w = self._pollable_worker()
|
||||
request_id = self._make_sending_req(w)
|
||||
w.transfer_topo.block_size_ratio.return_value = 1
|
||||
w._apply_prefix_caching = MagicMock(return_value=([[1]], [[1]]))
|
||||
w._compute_desc_ids = MagicMock(return_value=[0])
|
||||
w.dst_num_blocks = {w.engine_id: 8}
|
||||
w.dst_region_num_blocks[w.engine_id] = [8]
|
||||
w.dst_region_group_ids[w.engine_id] = [0]
|
||||
w.dst_uses_region_group_mapping[w.engine_id] = False
|
||||
w.nixl_wrapper.make_prepped_xfer.return_value = 101
|
||||
w.nixl_wrapper.transfer.side_effect = RuntimeError("submit")
|
||||
if release_fails:
|
||||
w.nixl_wrapper.release_xfer_handle.side_effect = [
|
||||
RuntimeError("release"),
|
||||
None,
|
||||
None,
|
||||
]
|
||||
|
||||
handle = w._xfer_blocks(
|
||||
read_spec=ReadSpec(0, [[1]], [[1]]),
|
||||
dst_engine_id=w.engine_id,
|
||||
request_id=request_id,
|
||||
remote_request_id="decode-request",
|
||||
local_xfer_side_handle=1,
|
||||
remote_xfer_side_handle=2,
|
||||
)
|
||||
assert handle == (101 if release_fails else None)
|
||||
w._sending_transfers[request_id] = [102]
|
||||
if handle is not None:
|
||||
w._sending_transfers[request_id].append(handle)
|
||||
w.nixl_wrapper.check_xfer_state.return_value = "DONE"
|
||||
|
||||
assert w.get_transfer_results().finished_sending == set()
|
||||
assert request_id in w._reqs_to_send
|
||||
assert request_id not in w._sending_transfers
|
||||
assert w._evict_finished_inbox.empty()
|
||||
|
||||
@pytest.mark.parametrize("first_state", ["ERR", "DONE"])
|
||||
def test_push_completion_after_expiry_is_not_reported_twice(self, first_state):
|
||||
"""A WRITE finishing after lease expiry must not complete the request again."""
|
||||
w = self._pollable_worker()
|
||||
request_id = self._make_sending_req(w)
|
||||
w.nixl_wrapper.check_xfer_state.side_effect = [
|
||||
first_state,
|
||||
"PROC",
|
||||
"PROC",
|
||||
"DONE",
|
||||
]
|
||||
assert w.get_transfer_results().finished_sending == set()
|
||||
|
||||
w.expected_consumer_notifications_by_req = {}
|
||||
w._reqs_to_send[request_id] = time.perf_counter() - 1
|
||||
assert w.get_transfer_results().finished_sending == {request_id}
|
||||
assert w._evict_finished_inbox.get_nowait() == request_id
|
||||
|
||||
assert w.get_transfer_results().finished_sending == set()
|
||||
assert request_id not in w._sending_transfers
|
||||
assert request_id not in w._send_failures
|
||||
assert w._evict_finished_inbox.empty()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- #
|
||||
# Negative / error-path tests #
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
@@ -8,7 +9,9 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash import speculator as dflash_speculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
|
||||
DFlashSpeculator,
|
||||
prepare_dflash_inputs,
|
||||
)
|
||||
|
||||
@@ -33,6 +36,7 @@ def _run_prepare(
|
||||
input_buffers = SimpleNamespace(
|
||||
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
|
||||
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
|
||||
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
|
||||
query_start_loc=torch.full(
|
||||
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
|
||||
),
|
||||
@@ -140,6 +144,114 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
|
||||
assert out.temperature[2].item() == 1.0
|
||||
assert out.seeds[2].item() == 17
|
||||
|
||||
assert not out.input_buffers.is_padding[:3].any()
|
||||
assert out.input_buffers.is_padding[3:].all()
|
||||
assert out.input_buffers.input_ids[3:].cpu().tolist() == [0] * 13
|
||||
assert out.input_buffers.positions[3:].cpu().tolist() == [0] * 13
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_compacts_noncontiguous_request_slots():
|
||||
device = torch.device("cuda")
|
||||
max_num_reqs = 4
|
||||
max_num_tokens = 16
|
||||
num_speculative_steps = 3
|
||||
input_buffers = SimpleNamespace(
|
||||
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
|
||||
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
|
||||
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
|
||||
query_start_loc=torch.full(
|
||||
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
|
||||
),
|
||||
seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device),
|
||||
)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
num_scheduled_tokens=np.array([4, 4], dtype=np.int32),
|
||||
positions=torch.tensor(
|
||||
[10, 11, 12, 13, 20, 21, 22, 23],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
),
|
||||
query_start_loc=torch.tensor([0, 4, 8], dtype=torch.int32, device=device),
|
||||
# Active batch rows are compact, while request state remains in slots 3 and 1.
|
||||
idx_mapping=torch.tensor([3, 1], dtype=torch.int32, device=device),
|
||||
)
|
||||
query_slot_mapping = torch.full(
|
||||
(max_num_tokens,), -2, dtype=torch.int64, device=device
|
||||
)
|
||||
context_positions = torch.full(
|
||||
(max_num_tokens,), -1, dtype=torch.int64, device=device
|
||||
)
|
||||
context_slot_mapping = torch.full(
|
||||
(max_num_tokens,), -2, dtype=torch.int64, device=device
|
||||
)
|
||||
sample_indices = torch.full(
|
||||
(max_num_reqs * num_speculative_steps,),
|
||||
-1,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
sample_pos = torch.full_like(sample_indices, -1)
|
||||
sample_idx_mapping = torch.full(
|
||||
sample_indices.shape, -1, dtype=torch.int32, device=device
|
||||
)
|
||||
temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device)
|
||||
seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device)
|
||||
input_temperature = torch.tensor(
|
||||
[0.0, 0.5, 0.0, 1.0], dtype=torch.float32, device=device
|
||||
)
|
||||
input_seeds = torch.tensor([0, 11, 0, 33], dtype=torch.int64, device=device)
|
||||
last_sampled = torch.tensor([0, 77, 0, 99], dtype=torch.int64, device=device)
|
||||
next_prefill_tokens = torch.zeros_like(last_sampled)
|
||||
block_table = torch.tensor(
|
||||
[[0, 0, 7, 8, 9, 10, 11, 12], [0, 0, 13, 14, 15, 16, 17, 18]],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
prepare_dflash_inputs(
|
||||
input_buffers,
|
||||
query_slot_mapping,
|
||||
context_positions,
|
||||
context_slot_mapping,
|
||||
sample_indices,
|
||||
sample_pos,
|
||||
sample_idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
input_batch,
|
||||
torch.tensor([1, 1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2, 1], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
block_table,
|
||||
4,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
123,
|
||||
num_speculative_steps,
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
128,
|
||||
sample_from_anchor=True,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Query rows follow compact batch order, but every persistent state lookup
|
||||
# follows idx_mapping instead of accidentally using the compact row index.
|
||||
assert input_buffers.input_ids[:6].cpu().tolist() == [99, 123, 123, 77, 123, 123]
|
||||
assert input_buffers.positions[:6].cpu().tolist() == [12, 13, 14, 23, 24, 25]
|
||||
assert sample_indices[:6].cpu().tolist() == [0, 1, 2, 3, 4, 5]
|
||||
assert sample_idx_mapping[:6].cpu().tolist() == [3, 3, 3, 1, 1, 1]
|
||||
assert temperature.cpu().tolist() == [0.0, 0.5, 0.0, 1.0]
|
||||
assert seeds.cpu().tolist() == [0, 11, 0, 33]
|
||||
assert not input_buffers.is_padding[:6].any()
|
||||
assert input_buffers.is_padding[6:].all()
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
|
||||
out = _run_prepare(
|
||||
@@ -174,3 +286,40 @@ def test_prepare_dflash_inputs_never_writes_the_null_block():
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
]
|
||||
|
||||
|
||||
def test_dflash_forward_context_receives_draft_padding_mask(monkeypatch):
|
||||
device = torch.device("cuda")
|
||||
input_buffers = SimpleNamespace(
|
||||
input_ids=torch.tensor([11, 12, 0, 0], dtype=torch.int32, device=device),
|
||||
positions=torch.tensor([7, 8, 0, 0], dtype=torch.int64, device=device),
|
||||
is_padding=torch.tensor([False, False, True, True], device=device),
|
||||
)
|
||||
observed = None
|
||||
|
||||
@contextmanager
|
||||
def fake_set_forward_context(*args, **kwargs):
|
||||
nonlocal observed
|
||||
observed = kwargs["is_padding"].clone()
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(
|
||||
dflash_speculator, "set_forward_context", fake_set_forward_context
|
||||
)
|
||||
speculator = SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
vllm_config=SimpleNamespace(),
|
||||
model=lambda **kwargs: kwargs["input_ids"],
|
||||
)
|
||||
|
||||
result = DFlashSpeculator._run_model(
|
||||
speculator,
|
||||
num_tokens=4,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=None,
|
||||
)
|
||||
|
||||
assert result.tolist() == [11, 12, 0, 0]
|
||||
assert observed is not None
|
||||
assert observed.tolist() == [False, False, True, True]
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import Platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
@@ -425,6 +426,62 @@ def test_select_common_block_size_no_valid_option():
|
||||
select_common_block_size(48, [backend_a, backend_b])
|
||||
|
||||
|
||||
def _mock_backend(supported: list, *, exact: bool = False):
|
||||
"""Backend accepting multiples of ``supported``, or only those exact sizes
|
||||
when ``exact`` (as CPU_MLA does)."""
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
|
||||
class _MockBackendCls(AttentionBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "MOCK_EXACT" if exact else "MOCK"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
return list(supported)
|
||||
|
||||
if exact:
|
||||
|
||||
@classmethod
|
||||
def supports_block_size(cls, block_size: int | None) -> bool:
|
||||
return block_size is None or block_size in supported
|
||||
|
||||
return _MockBackendCls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backends,expected",
|
||||
[
|
||||
# A lone backend keeps the default; MultipleOf(8) makes a
|
||||
# minimum-based regression return 8 instead of 16.
|
||||
([[MultipleOf(8)]], 16),
|
||||
# Sparse-MLA main backend (32 or 64) beside an indexer supporting only
|
||||
# 64: the main backend alone picks 32, which the indexer rejects at
|
||||
# select_common_block_size time.
|
||||
([[32, 64], [64]], 64),
|
||||
([[32, 64]], 32),
|
||||
# Neither divides the other, so only the LCM satisfies both.
|
||||
([[32], [48]], 96),
|
||||
],
|
||||
)
|
||||
def test_preferred_block_size_satisfies_every_backend(backends, expected):
|
||||
classes = [_mock_backend(s) for s in backends]
|
||||
assert Platform._preferred_block_size_for_backends(classes, 16, None) == expected
|
||||
|
||||
|
||||
def test_preferred_block_size_searches_past_an_exact_size_backend():
|
||||
# Extending greedily picks lcm(16, 32) = 32, which the exact backend
|
||||
# rejects; 96 is accepted by both.
|
||||
classes = [_mock_backend([16, 96], exact=True), _mock_backend([MultipleOf(32)])]
|
||||
assert Platform._preferred_block_size_for_backends(classes, 16, None) == 96
|
||||
|
||||
|
||||
def test_preferred_block_size_rejects_backends_with_no_common_size():
|
||||
classes = [_mock_backend([16], exact=True), _mock_backend([MultipleOf(64)])]
|
||||
with pytest.raises(ValueError, match="share no supported KV cache block size"):
|
||||
Platform._preferred_block_size_for_backends(classes, 16, None)
|
||||
|
||||
|
||||
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
|
||||
model = Mock()
|
||||
model.get_mm_lora_token_counts.side_effect = (
|
||||
|
||||
@@ -5,16 +5,35 @@
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.worker.gpu import pp_utils
|
||||
from vllm.v1.worker.gpu import model_runner, pp_utils
|
||||
|
||||
|
||||
def _batch(num_computed, prefill_len, num_scheduled):
|
||||
def _cuda_handler(max_sample_len=6):
|
||||
handler = object.__new__(pp_utils.PPHandler)
|
||||
handler.is_last_rank = True
|
||||
handler.disabled = False
|
||||
handler.max_sample_len = max_sample_len
|
||||
handler.last_rank = 1
|
||||
handler.broadcast_group = Mock()
|
||||
handler.device = torch.device("cuda")
|
||||
handler.main_stream = torch.cuda.current_stream()
|
||||
handler.broadcast_stream = torch.cuda.Stream()
|
||||
return handler
|
||||
|
||||
|
||||
def _batch(num_computed, prefill_len, num_scheduled, idx_mapping=None):
|
||||
num_reqs = len(num_computed)
|
||||
if idx_mapping is None:
|
||||
idx_mapping = list(range(num_reqs))
|
||||
return Mock(
|
||||
num_reqs=len(num_computed),
|
||||
num_reqs=num_reqs,
|
||||
num_computed_tokens_np=np.array(num_computed, dtype=np.int32),
|
||||
prefill_len_np=np.array(prefill_len, dtype=np.int32),
|
||||
num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32),
|
||||
idx_mapping=torch.tensor(idx_mapping, dtype=torch.int64),
|
||||
)
|
||||
|
||||
|
||||
@@ -81,3 +100,130 @@ def test_decode_row_ahead_of_a_prefill_chunk():
|
||||
|
||||
assert mask is not None
|
||||
assert mask.tolist() == [True, False]
|
||||
|
||||
|
||||
def test_disabled_handler_skips_broadcast_and_receive(monkeypatch):
|
||||
"""While disabled (warmup), neither side enqueues a broadcast op."""
|
||||
sent = []
|
||||
monkeypatch.setattr(
|
||||
pp_utils.torch.distributed,
|
||||
"broadcast",
|
||||
lambda *args, **kwargs: sent.append((args, kwargs)),
|
||||
)
|
||||
|
||||
handler = object.__new__(pp_utils.PPHandler)
|
||||
handler.set_disabled(True)
|
||||
|
||||
handler.is_last_rank = False
|
||||
assert handler.receive(Mock()) is False
|
||||
|
||||
handler.is_last_rank = True
|
||||
assert handler.broadcast(Mock(), Mock(), Mock(), Mock()) is None
|
||||
|
||||
assert sent == []
|
||||
|
||||
handler.set_disabled(False)
|
||||
assert handler.disabled is False
|
||||
|
||||
|
||||
def test_alloc_combined_keeps_unbind_views_16_byte_aligned():
|
||||
"""Triton specializes on pointer alignment: an unaligned `num_rejected`
|
||||
would compile a second `_post_update_kernel` variant at serving time,
|
||||
where the in-flight broadcast NCCL kernel can block the module load."""
|
||||
for num_reqs in range(1, 9):
|
||||
combined = pp_utils._alloc_combined(num_reqs, torch.device("cpu"))
|
||||
num_sampled, num_rejected = combined.unbind(dim=0)
|
||||
assert num_sampled.data_ptr() % 16 == 0
|
||||
assert num_rejected.data_ptr() % 16 == 0
|
||||
assert combined.shape[1] >= num_reqs
|
||||
|
||||
|
||||
def test_warmup_pp_decode_update_matches_serving_specialization(monkeypatch):
|
||||
"""The warmup launch must hit the same triton specialization as serving.
|
||||
|
||||
A mismatch means the first real ``update_pp_decode_requests`` recompiles
|
||||
mid-serving, where the in-flight broadcast NCCL kernel blocks the CUDA
|
||||
module load and deadlocks the pipeline.
|
||||
"""
|
||||
calls = []
|
||||
monkeypatch.setattr(model_runner, "post_update", lambda *args: calls.append(args))
|
||||
|
||||
runner = object.__new__(model_runner.GPUModelRunner)
|
||||
runner.device = torch.device("cpu")
|
||||
runner.pp_handler = Mock(max_sample_len=3)
|
||||
runner.req_states = Mock()
|
||||
|
||||
runner.warmup_pp_decode_update()
|
||||
|
||||
assert len(calls) == 1
|
||||
args = calls[0]
|
||||
idx_mapping, _, _, output_bin_counts = args[:4]
|
||||
sampled_tokens, num_sampled, num_rejected, query_start_loc = args[4:8]
|
||||
broadcast_drafts, draft_tokens_out = args[10:12]
|
||||
assert idx_mapping.tolist() == [-1] and idx_mapping.dtype == torch.int64
|
||||
assert output_bin_counts is None
|
||||
assert query_start_loc is None
|
||||
assert sampled_tokens.shape == (1, 3) and sampled_tokens.dtype == torch.int64
|
||||
assert num_sampled.dtype == torch.int32
|
||||
assert num_rejected.dtype == torch.int32
|
||||
# Spec-enabled PP handlers receive drafts over the broadcast; the warmup
|
||||
# must compile that specialization (non-None draft pointers) too.
|
||||
assert broadcast_drafts.shape == (1, 2) and broadcast_drafts.dtype == torch.int64
|
||||
assert draft_tokens_out is runner.req_states.draft_tokens
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA stream")
|
||||
def test_broadcast_pads_plain_sampler_rows_to_max_sample_len(monkeypatch):
|
||||
"""The wire shape must not depend on whether the batch carried drafts:
|
||||
the receiver always allocates [num_reqs, max_sample_len], and a NCCL
|
||||
broadcast with mismatched counts hangs the receiver."""
|
||||
sent = []
|
||||
monkeypatch.setattr(
|
||||
pp_utils.torch.distributed,
|
||||
"broadcast",
|
||||
lambda tensor, **kw: sent.append(tensor),
|
||||
)
|
||||
handler = _cuda_handler()
|
||||
batch = _batch(num_computed=[10], prefill_len=[8], num_scheduled=[1])
|
||||
|
||||
handler.broadcast(
|
||||
torch.zeros(1, 1, dtype=torch.int64, device="cuda"), # plain sampler
|
||||
torch.ones(1, dtype=torch.int32, device="cuda"),
|
||||
torch.zeros(1, dtype=torch.int32, device="cuda"),
|
||||
batch,
|
||||
)
|
||||
|
||||
assert sent[0].shape == (1, 6)
|
||||
assert sent[1].shape == (2, 4)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA stream")
|
||||
def test_broadcast_drafts_gathers_fresh_rows_from_the_table(monkeypatch):
|
||||
sent = []
|
||||
monkeypatch.setattr(
|
||||
pp_utils.torch.distributed,
|
||||
"broadcast",
|
||||
lambda tensor, **kw: sent.append(tensor),
|
||||
)
|
||||
handler = _cuda_handler()
|
||||
# The batch's single row maps to request-state row 2.
|
||||
batch = _batch(num_computed=[10], prefill_len=[8], num_scheduled=[1])
|
||||
batch.idx_mapping = torch.tensor([2], dtype=torch.int64, device="cuda")
|
||||
table = torch.arange(20, dtype=torch.int64, device="cuda").view(4, 5)
|
||||
|
||||
handler.broadcast_drafts(table, batch)
|
||||
|
||||
assert sent[0].shape == (1, 5)
|
||||
# The payload is a gather into a fresh tensor: propose() overwrites its
|
||||
# persistent buffer on the next step, possibly before this send completes.
|
||||
assert sent[0].data_ptr() != table.data_ptr()
|
||||
assert sent[0].cpu().tolist() == [table[2].cpu().tolist()]
|
||||
|
||||
# An all-prefill batch sends nothing (receive() enqueues nothing either).
|
||||
sent.clear()
|
||||
prefill_batch = _batch(num_computed=[0], prefill_len=[4096], num_scheduled=[448])
|
||||
prefill_batch.idx_mapping = prefill_batch.idx_mapping.cuda()
|
||||
handler.broadcast_drafts(table, prefill_batch)
|
||||
assert sent == []
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -3362,6 +3362,115 @@ class rocm_aiter_ops:
|
||||
rotary_dim=kernel_rotary_dim,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def fused_qk_norm_mrope_and_cache(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
num_heads_k: int,
|
||||
head_dim: int,
|
||||
is_neox: bool,
|
||||
mrope_section: list[int],
|
||||
is_interleaved: bool,
|
||||
rms_norm_eps: float,
|
||||
q_out: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
block_size: int,
|
||||
x: int,
|
||||
rotary_dim: int = 0,
|
||||
) -> None:
|
||||
from aiter.ops.fused_qk_norm_mrope_cache_quant import (
|
||||
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle,
|
||||
)
|
||||
|
||||
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
qkv.size(0),
|
||||
num_heads_q,
|
||||
num_heads_k,
|
||||
num_heads_k,
|
||||
head_dim,
|
||||
is_neox,
|
||||
mrope_section,
|
||||
is_interleaved,
|
||||
rms_norm_eps,
|
||||
q_out,
|
||||
k_cache,
|
||||
v_cache,
|
||||
slot_mapping,
|
||||
k_scale,
|
||||
v_scale,
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
block_size,
|
||||
x,
|
||||
rotary_dim,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def do_qk_norm_mrope_kvcache_update(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
num_heads_k: int,
|
||||
head_dim: int,
|
||||
is_neox: bool,
|
||||
mrope_section: list[int],
|
||||
is_interleaved: bool,
|
||||
rms_norm_eps: float,
|
||||
q_out: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
rotary_dim: int = 0,
|
||||
) -> None:
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||
value_cache = value_cache.view(current_platform.fp8_dtype())
|
||||
|
||||
rocm_aiter_ops.fused_qk_norm_mrope_and_cache(
|
||||
qkv=qkv,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
num_heads_q=num_heads_q,
|
||||
num_heads_k=num_heads_k,
|
||||
head_dim=head_dim,
|
||||
is_neox=is_neox,
|
||||
mrope_section=mrope_section,
|
||||
is_interleaved=is_interleaved,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
q_out=q_out,
|
||||
k_cache=key_cache,
|
||||
v_cache=value_cache,
|
||||
slot_mapping=slot_mapping,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
block_size=key_cache.shape[1],
|
||||
x=16 // key_cache.element_size(),
|
||||
rotary_dim=rotary_dim,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def triton_rope_and_cache(
|
||||
query: torch.Tensor,
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
ROTARY_OP = torch.ops._C.rotary_embedding.default
|
||||
FLASHINFER_ROTARY_OP = torch.ops.vllm.flashinfer_rotary_embedding.default
|
||||
MROPE_OP = torch.ops.vllm.mrope.default
|
||||
|
||||
QUANT_OPS: dict[QuantKey, OpOverload] = {
|
||||
kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa: E501
|
||||
@@ -162,6 +163,74 @@ class MatcherRotaryEmbedding(MatcherCustomOp):
|
||||
return result
|
||||
|
||||
|
||||
class MatcherMRotaryEmbedding(MatcherCustomOp):
|
||||
def __init__(
|
||||
self,
|
||||
is_neox: bool,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
mrope_section: tuple[int, int, int],
|
||||
mrope_interleaved: bool,
|
||||
enabled: bool | None = None,
|
||||
) -> None:
|
||||
if enabled is None:
|
||||
enabled = RotaryEmbedding.enabled()
|
||||
|
||||
super().__init__(enabled)
|
||||
self.is_neox = is_neox
|
||||
self.head_size = head_size
|
||||
self.rotary_dim = rotary_dim
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.q_size = self.num_heads * self.head_size
|
||||
self.kv_size = self.num_kv_heads * self.head_size
|
||||
self.mrope_section = mrope_section
|
||||
self.mrope_interleaved = mrope_interleaved
|
||||
|
||||
def inputs(self) -> list[torch.Tensor]:
|
||||
positions = self.empty_int64(3, 5)
|
||||
query = self.empty(5, self.q_size)
|
||||
key = self.empty(5, self.kv_size)
|
||||
cos_sin_cache = self.empty(4096, self.rotary_dim)
|
||||
return [positions, query, key, cos_sin_cache]
|
||||
|
||||
def forward_custom(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
result = auto_functionalized(
|
||||
MROPE_OP,
|
||||
positions=positions,
|
||||
query=query,
|
||||
key=key,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
head_size=self.head_size,
|
||||
rotary_dim=self.rotary_dim,
|
||||
mrope_section_t=self.mrope_section[0],
|
||||
mrope_section_h=self.mrope_section[1],
|
||||
mrope_section_w=self.mrope_section[2],
|
||||
mrope_interleaved=self.mrope_interleaved,
|
||||
is_neox_style=self.is_neox,
|
||||
)
|
||||
return result[1], result[2]
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
raise NotImplementedError(
|
||||
"MRoPE fusion requires the rotary_embedding custom op"
|
||||
)
|
||||
|
||||
|
||||
class MatcherRMSNormGated(MatcherCustomOp):
|
||||
"""Matches RMSNormGated with norm_before_gate=True and group_size=None."""
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import ParamSpec
|
||||
|
||||
import torch
|
||||
@@ -19,12 +19,19 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
Attention,
|
||||
get_attention_context,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import (
|
||||
_USE_LAYERNAME,
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
|
||||
from ..inductor_pass import enable_fake_mode
|
||||
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
from .matcher_utils import MatcherRotaryEmbedding
|
||||
from .matcher_utils import MROPE_OP, MatcherMRotaryEmbedding, MatcherRotaryEmbedding
|
||||
from .rms_quant_fusion import empty_bf16, empty_fp32, empty_i64
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -51,8 +58,9 @@ def fused_qk_norm_rope_and_unified_kv_cache_update_impl(
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
layer_name: str = "",
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
|
||||
if layer_slot_mapping is not None:
|
||||
attn_layer.impl.do_qk_norm_rope_kvcache_update(
|
||||
@@ -87,7 +95,7 @@ def fused_qk_norm_rope_and_unified_kv_cache_update_fake(
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
layer_name: str = "",
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
|
||||
|
||||
@@ -100,6 +108,76 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def fused_qk_norm_mrope_and_unified_kv_cache_update_impl(
|
||||
q_out: torch.Tensor,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
mrope_section_t: int,
|
||||
mrope_section_h: int,
|
||||
mrope_section_w: int,
|
||||
is_interleaved: bool,
|
||||
rotary_dim: int,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
|
||||
if layer_slot_mapping is not None:
|
||||
if positions.ndim == 1:
|
||||
positions = positions.unsqueeze(0).expand(3, -1).contiguous()
|
||||
attn_layer.impl.do_qk_norm_mrope_kvcache_update(
|
||||
attn_layer,
|
||||
qkv,
|
||||
q_out,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
(mrope_section_t, mrope_section_h, mrope_section_w),
|
||||
is_interleaved,
|
||||
rotary_dim,
|
||||
kv_cache,
|
||||
layer_slot_mapping,
|
||||
)
|
||||
else:
|
||||
q_out.zero_()
|
||||
|
||||
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
|
||||
|
||||
|
||||
def fused_qk_norm_mrope_and_unified_kv_cache_update_fake(
|
||||
q_out: torch.Tensor,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
mrope_section_t: int,
|
||||
mrope_section_h: int,
|
||||
mrope_section_w: int,
|
||||
is_interleaved: bool,
|
||||
rotary_dim: int,
|
||||
layer_name: LayerNameType,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_qk_norm_mrope_and_unified_kv_cache_update",
|
||||
op_func=fused_qk_norm_mrope_and_unified_kv_cache_update_impl,
|
||||
mutates_args=["q_out"],
|
||||
fake_impl=fused_qk_norm_mrope_and_unified_kv_cache_update_fake,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern: QK-norm + RoPE + unified_kv_cache_update
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -142,19 +220,27 @@ class QkNormRopeKvCachePattern:
|
||||
self.eps = eps
|
||||
self.is_neox = is_neox
|
||||
self.quant_query = quant_query
|
||||
self.encoded_layer_name = _encode_layer_name(self.layer_name)
|
||||
self.query_quant_group_shape = (
|
||||
layer.query_quant.group_shape
|
||||
if layer.query_quant is not None
|
||||
else GroupShape.PER_TENSOR
|
||||
)
|
||||
|
||||
self.q_size = self.num_heads * self.head_size
|
||||
self.k_size = self.num_kv_heads * self.head_size
|
||||
self.v_size = self.num_kv_heads * self.head_size_v
|
||||
|
||||
self.rope_matcher = MatcherRotaryEmbedding(
|
||||
is_neox=is_neox,
|
||||
head_size=self.head_size,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
self.rope_matcher: MatcherRotaryEmbedding | MatcherMRotaryEmbedding = (
|
||||
MatcherRotaryEmbedding(
|
||||
is_neox=is_neox,
|
||||
head_size=self.head_size,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
)
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
def get_inputs(self) -> list:
|
||||
T = 5
|
||||
L = 4096
|
||||
qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size)
|
||||
@@ -166,8 +252,13 @@ class QkNormRopeKvCachePattern:
|
||||
if self.quant_query:
|
||||
q_scale = empty_fp32(1)
|
||||
inputs += [q_scale]
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(self.encoded_layer_name)
|
||||
return inputs
|
||||
|
||||
def _get_layer_name(self, layer_name: LayerNameType | None) -> LayerNameType:
|
||||
return self.encoded_layer_name if layer_name is None else layer_name
|
||||
|
||||
def pattern_non_fp8_quant_query(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
@@ -175,6 +266,7 @@ class QkNormRopeKvCachePattern:
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
layer_name: LayerNameType | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
|
||||
@@ -190,7 +282,9 @@ class QkNormRopeKvCachePattern:
|
||||
q_rope = q_rope.view(-1, self.num_heads, self.head_size)
|
||||
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size_v)
|
||||
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
|
||||
dummy = torch.ops.vllm.unified_kv_cache_update(
|
||||
k_rope, v, self._get_layer_name(layer_name)
|
||||
)
|
||||
return dummy, q_rope, k_rope, v
|
||||
|
||||
def replacement_non_fp8_quant_query(
|
||||
@@ -200,6 +294,7 @@ class QkNormRopeKvCachePattern:
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
layer_name: LayerNameType | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q_out = torch.empty(
|
||||
qkv.shape[0],
|
||||
@@ -228,10 +323,49 @@ class QkNormRopeKvCachePattern:
|
||||
rms_norm_eps=self.eps,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
layer_name=self.layer_name,
|
||||
layer_name=self._get_layer_name(layer_name),
|
||||
)
|
||||
return results[0], results[1], results[2], v
|
||||
|
||||
def _quantize_query(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if self.query_quant_group_shape.is_per_tensor():
|
||||
q_out = torch.empty_like(query, dtype=current_platform.fp8_dtype())
|
||||
q_quant = auto_functionalized(
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
|
||||
out=q_out,
|
||||
x=query,
|
||||
scale=q_scale,
|
||||
is_dynamic=False,
|
||||
)
|
||||
# The AITER op's schema marks scale mutable, so preserve its
|
||||
# functionalized write-back as an output of the matched region.
|
||||
return q_quant[1], q_quant[2]
|
||||
|
||||
# QuantFP8.forward_hip falls back to the generic static op for the
|
||||
# historical per-head shape (-1, block_size). This also covers the
|
||||
# degenerate one-local-KV-head case where a scalar scale is represented
|
||||
# as (-1, q_size), rather than changing Attention's classification.
|
||||
q_out = torch.empty(
|
||||
query.shape,
|
||||
device=query.device,
|
||||
dtype=current_platform.fp8_dtype(),
|
||||
)
|
||||
q_quant = auto_functionalized(
|
||||
torch.ops._C.static_scaled_fp8_quant.default,
|
||||
result=q_out,
|
||||
input=query,
|
||||
scale=q_scale,
|
||||
group_shape=[
|
||||
self.query_quant_group_shape.row,
|
||||
self.query_quant_group_shape.col,
|
||||
],
|
||||
)
|
||||
return q_quant[1], None
|
||||
|
||||
def pattern_fp8_quant_query(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
@@ -240,7 +374,8 @@ class QkNormRopeKvCachePattern:
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
layer_name: LayerNameType | None = None,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
|
||||
q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps)
|
||||
@@ -251,24 +386,15 @@ class QkNormRopeKvCachePattern:
|
||||
k_flat = k_normed.view(-1, self.k_size)
|
||||
|
||||
q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache)
|
||||
# Match the quant-query op Attention.forward inserts (fp8 KV + UNIFIED).
|
||||
# Explicit auto_functionalized (out=[1]) keeps the quant node in the pattern.
|
||||
q_out = torch.empty_like(q_rope, dtype=current_platform.fp8_dtype())
|
||||
q_quant = auto_functionalized(
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
|
||||
out=q_out,
|
||||
x=q_rope,
|
||||
scale=q_scale,
|
||||
is_dynamic=False,
|
||||
)
|
||||
# `scale` is mutable: its copy_ write-back to _q_scale bumps the mutation
|
||||
# region, so keep q flat (a reshape lands past the barrier and won't match).
|
||||
q_rope_fp8 = q_quant[1]
|
||||
q_scale_out = q_quant[2]
|
||||
q_rope_fp8, q_scale_out = self._quantize_query(q_rope, q_scale)
|
||||
|
||||
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size_v)
|
||||
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
|
||||
dummy = torch.ops.vllm.unified_kv_cache_update(
|
||||
k_rope, v, self._get_layer_name(layer_name)
|
||||
)
|
||||
if q_scale_out is None:
|
||||
return dummy, q_rope_fp8, k_rope, v
|
||||
return dummy, q_rope_fp8, k_rope, v, q_scale_out
|
||||
|
||||
def replacement_fp8_quant_query(
|
||||
@@ -279,7 +405,8 @@ class QkNormRopeKvCachePattern:
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
layer_name: LayerNameType | None = None,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
q_out = torch.empty(
|
||||
qkv.shape[0],
|
||||
self.num_heads,
|
||||
@@ -307,22 +434,14 @@ class QkNormRopeKvCachePattern:
|
||||
rms_norm_eps=self.eps,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
layer_name=self.layer_name,
|
||||
layer_name=self._get_layer_name(layer_name),
|
||||
)
|
||||
# Re-apply the quant on the kernel's bf16 q_out; fused op does not quant q.
|
||||
# Same explicit auto_functionalized form as the pattern: [1] = quantized
|
||||
# q, [2] = scale (returned so the buffer-writeback use is preserved).
|
||||
# Re-apply the same quant form on the kernel's bf16 q_out; the fused
|
||||
# operation does not quantize Q.
|
||||
q_fp8_flat = results[1].view(-1, self.q_size)
|
||||
q_fp8_out = torch.empty_like(q_fp8_flat, dtype=current_platform.fp8_dtype())
|
||||
q_requant = auto_functionalized(
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
|
||||
out=q_fp8_out,
|
||||
x=q_fp8_flat,
|
||||
scale=q_scale,
|
||||
is_dynamic=False,
|
||||
)
|
||||
q_fp8 = q_requant[1] # flat to mirror the pattern (see note above)
|
||||
q_scale_out = q_requant[2]
|
||||
q_fp8, q_scale_out = self._quantize_query(q_fp8_flat, q_scale)
|
||||
if q_scale_out is None:
|
||||
return results[0], q_fp8, results[2], v
|
||||
return results[0], q_fp8, results[2], v, q_scale_out
|
||||
|
||||
@staticmethod
|
||||
@@ -345,6 +464,9 @@ class QkNormRopeKvCachePattern:
|
||||
|
||||
view_to_reshape(gm)
|
||||
|
||||
def _extra_check(self, match: pm.Match) -> bool:
|
||||
return True
|
||||
|
||||
def _register(self, pattern, replacement, pm_pass) -> None:
|
||||
trace_fn = QkNormRopeKvCachePattern.wrap_trace_fn(
|
||||
pm.fwd_only,
|
||||
@@ -370,40 +492,338 @@ class QkNormRopeKvCachePattern:
|
||||
inputs,
|
||||
trace_fn,
|
||||
pm_pass,
|
||||
extra_check=self._extra_check,
|
||||
search_fn_pattern=search_fn_pattern,
|
||||
)
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
# make_fx counts `self` in bound-method code params; wrap as plain fns.
|
||||
# Distinct names per branch so mypy doesn't see one name, two signatures.
|
||||
if self.quant_query:
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def pattern_q(qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale):
|
||||
return self.pattern_fp8_quant_query(
|
||||
def pattern_q_with_layer(
|
||||
qkv,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
q_scale,
|
||||
layer_name,
|
||||
):
|
||||
return self.pattern_fp8_quant_query(
|
||||
qkv,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
q_scale,
|
||||
layer_name,
|
||||
)
|
||||
|
||||
def replacement_q_with_layer(
|
||||
qkv,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
q_scale,
|
||||
layer_name,
|
||||
):
|
||||
return self.replacement_fp8_quant_query(
|
||||
qkv,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
q_scale,
|
||||
layer_name,
|
||||
)
|
||||
|
||||
self._register(pattern_q_with_layer, replacement_q_with_layer, pm_pass)
|
||||
else:
|
||||
|
||||
def pattern_q_without_layer(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
|
||||
)
|
||||
):
|
||||
return self.pattern_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
|
||||
)
|
||||
|
||||
def replacement_q(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
|
||||
):
|
||||
return self.replacement_fp8_quant_query(
|
||||
def replacement_q_without_layer(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
|
||||
)
|
||||
):
|
||||
return self.replacement_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
|
||||
)
|
||||
|
||||
self._register(pattern_q, replacement_q, pm_pass)
|
||||
self._register(
|
||||
pattern_q_without_layer, replacement_q_without_layer, pm_pass
|
||||
)
|
||||
else:
|
||||
if _USE_LAYERNAME:
|
||||
|
||||
def pattern_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
|
||||
return self.pattern_non_fp8_quant_query(
|
||||
def pattern_noq_with_layer(
|
||||
qkv,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
layer_name,
|
||||
):
|
||||
return self.pattern_non_fp8_quant_query(
|
||||
qkv,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
layer_name,
|
||||
)
|
||||
|
||||
def replacement_noq_with_layer(
|
||||
qkv,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
layer_name,
|
||||
):
|
||||
return self.replacement_non_fp8_quant_query(
|
||||
qkv,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
layer_name,
|
||||
)
|
||||
|
||||
self._register(
|
||||
pattern_noq_with_layer, replacement_noq_with_layer, pm_pass
|
||||
)
|
||||
else:
|
||||
|
||||
def pattern_noq_without_layer(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache
|
||||
):
|
||||
return self.pattern_non_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache
|
||||
)
|
||||
|
||||
def replacement_noq_without_layer(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache
|
||||
):
|
||||
return self.replacement_non_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache
|
||||
)
|
||||
|
||||
self._register(
|
||||
pattern_noq_without_layer,
|
||||
replacement_noq_without_layer,
|
||||
pm_pass,
|
||||
)
|
||||
|
||||
def replacement_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
|
||||
return self.replacement_non_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache
|
||||
)
|
||||
|
||||
self._register(pattern_noq, replacement_noq, pm_pass)
|
||||
class QkNormMRopeKvCachePattern(QkNormRopeKvCachePattern):
|
||||
FUSED_OP = torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer: Attention,
|
||||
eps: float,
|
||||
is_neox: bool,
|
||||
quant_query: bool,
|
||||
rotary_dim: int,
|
||||
mrope_section: tuple[int, int, int],
|
||||
is_interleaved: bool,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
layer=layer,
|
||||
eps=eps,
|
||||
is_neox=is_neox,
|
||||
quant_query=quant_query,
|
||||
)
|
||||
self.rotary_dim = rotary_dim
|
||||
self.mrope_section = mrope_section
|
||||
self.is_interleaved = is_interleaved
|
||||
self.rope_matcher = MatcherMRotaryEmbedding(
|
||||
is_neox=is_neox,
|
||||
head_size=self.head_size,
|
||||
rotary_dim=rotary_dim,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
mrope_section=mrope_section,
|
||||
mrope_interleaved=is_interleaved,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
T = 5
|
||||
L = 4096
|
||||
qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size)
|
||||
positions = empty_i64(3, T)
|
||||
q_weight = empty_bf16(1, self.head_size)
|
||||
k_weight = empty_bf16(1, self.head_size)
|
||||
cos_sin_cache = empty_bf16(L, self.rotary_dim)
|
||||
inputs = [qkv, positions, q_weight, k_weight, cos_sin_cache]
|
||||
if self.quant_query:
|
||||
inputs.append(empty_fp32(1))
|
||||
if _USE_LAYERNAME:
|
||||
inputs.append(self.encoded_layer_name)
|
||||
return inputs
|
||||
|
||||
def _unfused_kv_views(self, qkv: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# The supported backend reads K/V from the cache after the fused write.
|
||||
# Keep view-compatible placeholders for the attention custom-op ABI.
|
||||
_, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
k = k.view(qkv.shape[0], self.num_kv_heads, self.head_size)
|
||||
v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v)
|
||||
return k, v
|
||||
|
||||
def _fused_results(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
layer_name: LayerNameType | None,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
q_out = torch.empty(
|
||||
qkv.shape[0],
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
device=qkv.device,
|
||||
dtype=qkv.dtype,
|
||||
)
|
||||
return auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
q_out=q_out,
|
||||
qkv=qkv,
|
||||
positions=positions,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
rms_norm_eps=self.eps,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
mrope_section_t=self.mrope_section[0],
|
||||
mrope_section_h=self.mrope_section[1],
|
||||
mrope_section_w=self.mrope_section[2],
|
||||
is_interleaved=self.is_interleaved,
|
||||
rotary_dim=self.rotary_dim,
|
||||
layer_name=self._get_layer_name(layer_name),
|
||||
)
|
||||
|
||||
def replacement_non_fp8_quant_query(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
layer_name: LayerNameType | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
k, v = self._unfused_kv_views(qkv)
|
||||
results = self._fused_results(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, layer_name
|
||||
)
|
||||
return results[0], results[1], k, v
|
||||
|
||||
def replacement_fp8_quant_query(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
layer_name: LayerNameType | None = None,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
k, v = self._unfused_kv_views(qkv)
|
||||
results = self._fused_results(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, layer_name
|
||||
)
|
||||
q_flat = results[1].view(-1, self.q_size)
|
||||
q_fp8, q_scale_out = self._quantize_query(q_flat, q_scale)
|
||||
if q_scale_out is None:
|
||||
return results[0], q_fp8, k, v
|
||||
return results[0], q_fp8, k, v, q_scale_out
|
||||
|
||||
def _extra_check(self, match: pm.Match) -> bool:
|
||||
expected = {
|
||||
"head_size": self.head_size,
|
||||
"rotary_dim": self.rotary_dim,
|
||||
"mrope_section_t": self.mrope_section[0],
|
||||
"mrope_section_h": self.mrope_section[1],
|
||||
"mrope_section_w": self.mrope_section[2],
|
||||
"mrope_interleaved": self.is_interleaved,
|
||||
"is_neox_style": self.is_neox,
|
||||
}
|
||||
mrope_node = next(
|
||||
(node for node in match.nodes if node.args and node.args[0] == MROPE_OP),
|
||||
None,
|
||||
)
|
||||
if mrope_node is None or any(
|
||||
mrope_node.kwargs.get(name) != value for name, value in expected.items()
|
||||
):
|
||||
return False
|
||||
|
||||
# The fused AITER MRoPE entrypoint writes rotated K directly to the
|
||||
# cache and does not expose a BF16 K result. Returning the raw K view is
|
||||
# valid only for decoder unified attention, whose backend consumes K
|
||||
# from that cache. Reject graphs that observe K in any other way.
|
||||
output_nodes = match.output_nodes()
|
||||
if len(output_nodes) < 3 or output_nodes[2] is None:
|
||||
return False
|
||||
matched_nodes = set(match.nodes)
|
||||
attention_op = torch.ops.vllm.unified_attention_with_output.default
|
||||
|
||||
def is_unified_attention_user(user: fx.Node) -> bool:
|
||||
return user.op == "call_function" and (
|
||||
user.target == attention_op
|
||||
or (user.args and user.args[0] == attention_op)
|
||||
)
|
||||
|
||||
return all(
|
||||
user in matched_nodes or is_unified_attention_user(user)
|
||||
for user in output_nodes[2].users
|
||||
)
|
||||
|
||||
|
||||
def _discover_mrope_configs(
|
||||
config: VllmConfig,
|
||||
) -> tuple[tuple[tuple[int, int, int], bool], ...]:
|
||||
model_config = config.model_config
|
||||
hf_configs = (
|
||||
getattr(model_config, "hf_text_config", None),
|
||||
getattr(model_config, "hf_config", None),
|
||||
)
|
||||
configs: set[tuple[tuple[int, int, int], bool]] = set()
|
||||
for hf_config in hf_configs:
|
||||
if hf_config is None:
|
||||
continue
|
||||
rope_parameters = getattr(hf_config, "rope_parameters", None)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = getattr(hf_config, "rope_scaling", None)
|
||||
if not isinstance(rope_parameters, Mapping):
|
||||
continue
|
||||
section = rope_parameters.get("mrope_section")
|
||||
if not isinstance(section, (list, tuple)) or len(section) != 3:
|
||||
continue
|
||||
try:
|
||||
parsed_section = (
|
||||
int(section[0]),
|
||||
int(section[1]),
|
||||
int(section[2]),
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if any(value <= 0 for value in parsed_section):
|
||||
continue
|
||||
configs.add(
|
||||
(
|
||||
parsed_section,
|
||||
bool(rope_parameters.get("mrope_interleaved", False)),
|
||||
)
|
||||
)
|
||||
return tuple(sorted(configs))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -412,7 +832,7 @@ class QkNormRopeKvCachePattern:
|
||||
|
||||
|
||||
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
"""Fuse QK-norm + RoPE/MRoPE + KV cache update into an AITER HIP kernel.
|
||||
|
||||
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
|
||||
attention layers that support the combined operation, eliminating two
|
||||
@@ -429,22 +849,33 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
|
||||
cc = config.compilation_config
|
||||
self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num
|
||||
self.mrope_configs = (
|
||||
_discover_mrope_configs(config)
|
||||
if cc.is_custom_op_enabled("rotary_embedding")
|
||||
else ()
|
||||
)
|
||||
|
||||
dtype = config.model_config.dtype
|
||||
if dtype not in (torch.bfloat16, torch.float16):
|
||||
logger.warning_once(
|
||||
"QK Norm+RoPE+KVCache fusion not enabled: unsupported dtype %s", dtype
|
||||
"QK Norm+RoPE/MRoPE+KVCache fusion not enabled: unsupported dtype %s",
|
||||
dtype,
|
||||
)
|
||||
return
|
||||
|
||||
attn_layers = get_layers_from_vllm_config(config, Attention)
|
||||
|
||||
for _, layer in attn_layers.items():
|
||||
if not layer.impl.fused_qk_norm_rope_kvcache_supported():
|
||||
supports_rope = layer.impl.fused_qk_norm_rope_kvcache_supported()
|
||||
supports_mrope = (
|
||||
bool(self.mrope_configs)
|
||||
and layer.impl.fused_qk_norm_mrope_kvcache_supported()
|
||||
)
|
||||
if not supports_rope and not supports_mrope:
|
||||
continue
|
||||
if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS:
|
||||
logger.warning_once(
|
||||
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
|
||||
"QK Norm+RoPE/MRoPE+KVCache fusion not enabled for a layer: "
|
||||
"head_size=%d is not supported by the "
|
||||
"fused_qk_norm_rope_cache_pts_quant_shuffle kernel "
|
||||
"(supported: %s). Falling back to the unfused path.",
|
||||
@@ -455,22 +886,71 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
if layer.head_size_v != layer.head_size:
|
||||
# The fused kernel uses a single head_dim for q/k/v.
|
||||
logger.warning_once(
|
||||
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
|
||||
"QK Norm+RoPE/MRoPE+KVCache fusion not enabled for a layer: "
|
||||
"head_size_v=%d differs from head_size=%d, which the fused "
|
||||
"kernel does not support. Falling back to the unfused path.",
|
||||
layer.head_size_v,
|
||||
layer.head_size,
|
||||
)
|
||||
continue
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
for neox in [True, False]:
|
||||
for quant_q in [False, True]:
|
||||
QkNormRopeKvCachePattern(
|
||||
layer=layer,
|
||||
eps=epsilon,
|
||||
is_neox=neox,
|
||||
quant_query=quant_q,
|
||||
).register(self.patterns)
|
||||
if supports_rope:
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
for neox in [True, False]:
|
||||
for quant_q in [False, True]:
|
||||
QkNormRopeKvCachePattern(
|
||||
layer=layer,
|
||||
eps=epsilon,
|
||||
is_neox=neox,
|
||||
quant_query=quant_q,
|
||||
).register(self.patterns)
|
||||
|
||||
if supports_mrope:
|
||||
for section, is_interleaved in self.mrope_configs:
|
||||
rotary_dim = 2 * sum(section)
|
||||
if rotary_dim > layer.head_size:
|
||||
logger.warning_once(
|
||||
"QK Norm+MRoPE+KVCache fusion not enabled for a "
|
||||
"layer: rotary_dim=%d exceeds head_size=%d.",
|
||||
rotary_dim,
|
||||
layer.head_size,
|
||||
)
|
||||
continue
|
||||
vector_width = layer.head_size // 32
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
for neox in [True, False]:
|
||||
# Each warp lane processes `vector_width` adjacent
|
||||
# values. NeoX additionally exchanges vectors
|
||||
# across the half-rotary boundary.
|
||||
rotary_alignment = vector_width * (2 if neox else 1)
|
||||
if rotary_dim % rotary_alignment != 0:
|
||||
logger.warning_once(
|
||||
"QK Norm+MRoPE+KVCache fusion not enabled "
|
||||
"for a layer/style: rotary_dim=%d must be "
|
||||
"divisible by %d for head_size=%d and "
|
||||
"is_neox=%s.",
|
||||
rotary_dim,
|
||||
rotary_alignment,
|
||||
layer.head_size,
|
||||
neox,
|
||||
)
|
||||
continue
|
||||
for quant_q in [False, True]:
|
||||
QkNormMRopeKvCachePattern(
|
||||
layer=layer,
|
||||
eps=epsilon,
|
||||
is_neox=neox,
|
||||
quant_query=quant_q,
|
||||
rotary_dim=rotary_dim,
|
||||
mrope_section=section,
|
||||
is_interleaved=is_interleaved,
|
||||
).register(self.patterns)
|
||||
|
||||
# Opaque LayerName is a pattern wildcard on torch >= 2.11, so
|
||||
# homogeneous attention layers produce duplicate search patterns.
|
||||
# As in the other LayerName-aware fusion passes, one supported
|
||||
# layer registers all shape/style variants for the model.
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
self.dump_patterns(config, self.patterns)
|
||||
|
||||
@@ -478,8 +958,8 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
self.matched_count = self.patterns.apply(graph)
|
||||
logger.info(
|
||||
"QK-Norm+RoPE+KVCache fusion: replaced %s pattern(s) "
|
||||
"with AITER fused_qk_norm_rope_cache_pts_quant_shuffle",
|
||||
"QK-Norm+RoPE/MRoPE+KVCache fusion: replaced %s pattern(s) "
|
||||
"with an AITER fused attention-prologue kernel",
|
||||
self.matched_count,
|
||||
)
|
||||
|
||||
@@ -487,4 +967,10 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
return compile_range.end <= self.max_token_num
|
||||
|
||||
def uuid(self) -> str:
|
||||
return VllmInductorPass.hash_source(self, QkNormRopeKvCachePattern)
|
||||
return VllmInductorPass.hash_source(
|
||||
self,
|
||||
QkNormRopeKvCachePattern,
|
||||
QkNormMRopeKvCachePattern,
|
||||
_discover_mrope_configs,
|
||||
repr(self.mrope_configs),
|
||||
)
|
||||
|
||||
@@ -154,15 +154,15 @@ class PassConfig:
|
||||
fuse_rope_kvcache: bool = None # type: ignore[assignment]
|
||||
"""Fuse the QK rope + KV cache ops."""
|
||||
fuse_qk_norm_rope_kvcache: bool = Field(default=None) # type: ignore[assignment]
|
||||
"""Fuse QK RMSNorm + RoPE + KV cache update into a single AITER HIP
|
||||
"""Fuse QK RMSNorm + RoPE/MRoPE + KV cache update into an AITER HIP
|
||||
kernel. Supersedes both enable_qk_norm_rope_fusion and fuse_rope_kvcache
|
||||
for layers that support it. Auto-enabled at O1+ on ROCm for models
|
||||
with QK-norm (e.g. Qwen3-MoE)."""
|
||||
for layers that support it. Auto-enabled at O2+ on ROCm for models
|
||||
with QK-norm (e.g. Qwen3-MoE and Qwen3-VL-class architectures)."""
|
||||
|
||||
rope_kvcache_fusion_max_token_num: int = 256
|
||||
"""The threshold for ROCm AITER RoPE+KVCache fusion e.g. for small batch decode.
|
||||
Larger batch sizes e.g. during prefill will use the unfused kernels.
|
||||
Also applies to the fused QK-Norm+RoPE+KVCache pass.
|
||||
Also applies to the fused QK-Norm+RoPE/MRoPE+KVCache pass.
|
||||
"""
|
||||
|
||||
fi_allreduce_fusion_max_size_mb: float | None = None
|
||||
|
||||
+50
-1
@@ -242,7 +242,7 @@ def enable_mla_dual_rms_norm_fusion(cfg: "VllmConfig") -> bool:
|
||||
|
||||
|
||||
def enable_qk_norm_rope_kvcache(cfg: "VllmConfig") -> bool:
|
||||
"""Enable fused QK-norm + RoPE + KV cache update on ROCm with AITER."""
|
||||
"""Enable fused QK-norm + RoPE/MRoPE + KV cache update with AITER."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_enabled():
|
||||
@@ -776,6 +776,45 @@ class VllmConfig:
|
||||
architectures = set(model_config.architectures)
|
||||
return bool(architectures & default_breakable_cudagraph_architectures())
|
||||
|
||||
def _uses_breakable_cudagraph_for_batch_invariance(self) -> bool:
|
||||
"""Avoid freezing runtime-M tile lookup in compiled forward (#54243).
|
||||
Breakable graphs look up tuned bf16, unquantized qkv/o/gate_up/down tiles
|
||||
at capture; lm_head runs outside compiled forward and does not benefit."""
|
||||
from vllm.model_executor.determinism import batch_invariant_configs as bi
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
model = self.model_config
|
||||
if (
|
||||
not envs.VLLM_BATCH_INVARIANT
|
||||
or model is None
|
||||
or model.enforce_eager
|
||||
or model.dtype != torch.bfloat16
|
||||
or model.quantization is not None
|
||||
or not current_platform.is_cuda()
|
||||
# Sequence parallelism / async TP are torch.compile passes.
|
||||
or self.compilation_config.pass_config.enable_sp
|
||||
or self.compilation_config.pass_config.fuse_gemm_comms
|
||||
):
|
||||
return False
|
||||
family = bi._get_tuned_matmul_arch_family(
|
||||
current_platform.get_device_capability()
|
||||
)
|
||||
if family is None or family not in bi._BATCH_INVARIANT_MATMUL_TUNED_CONFIGS:
|
||||
return False
|
||||
table = bi._BATCH_INVARIANT_MATMUL_TUNED_CONFIGS[family]
|
||||
parallel = self.parallel_config
|
||||
tp = parallel.tensor_parallel_size
|
||||
hidden = model.get_hidden_size()
|
||||
head = model.get_head_size()
|
||||
heads = model.get_num_attention_heads(parallel)
|
||||
kv_heads = model.get_num_kv_heads(parallel)
|
||||
shapes = [((heads + 2 * kv_heads) * head, hidden), (hidden, heads * head)]
|
||||
intermediate = getattr(model.hf_text_config, "intermediate_size", None)
|
||||
# Per-layer sizes (e.g. Gemma3n) are not modeled.
|
||||
if isinstance(intermediate, int):
|
||||
shapes += [(2 * intermediate // tp, hidden), (hidden, intermediate // tp)]
|
||||
return any(shape in table for shape in shapes)
|
||||
|
||||
def _maybe_enable_breakable_cudagraph(self) -> bool:
|
||||
if (
|
||||
"VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ
|
||||
@@ -786,6 +825,16 @@ class VllmConfig:
|
||||
"Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. "
|
||||
"Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out."
|
||||
)
|
||||
elif (
|
||||
"VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ
|
||||
and envs.VLLM_BATCH_INVARIANT
|
||||
and self._uses_breakable_cudagraph_for_batch_invariance()
|
||||
):
|
||||
os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
|
||||
logger.info_once(
|
||||
"VLLM_BATCH_INVARIANT=1: auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. "
|
||||
"Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out."
|
||||
)
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
is_breakable_cudagraph_enabled,
|
||||
|
||||
@@ -103,6 +103,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# (``_pop_done_transfers``); guarded by
|
||||
# ``_sending_transfers_lock``.
|
||||
self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list)
|
||||
self._send_failures: set[ReqId] = set()
|
||||
self._sending_transfers_lock = threading.Lock()
|
||||
|
||||
# Writer-thread owned matching state.
|
||||
@@ -162,6 +163,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self._sending_transfers.clear()
|
||||
self._send_failures.clear()
|
||||
super().shutdown()
|
||||
|
||||
# --- Engine-main-thread entry point -------------------------------- #
|
||||
@@ -728,6 +730,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# don't have a ``_recving_metadata`` entry to invalidate, so
|
||||
# we just release the handle and let the engine reschedule
|
||||
# via the lease / watchdog.
|
||||
with self._sending_transfers_lock:
|
||||
self._send_failures.add(request_id)
|
||||
if not self._handle_failed_transfer(request_id, handle):
|
||||
return handle
|
||||
return None
|
||||
@@ -811,13 +815,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
done_pushing, failed_pushing = self._pop_done_transfers(
|
||||
self._sending_transfers
|
||||
)
|
||||
# A failed send must never be reported as done: its blocks
|
||||
# are freed via the lease / watchdog instead.
|
||||
done_pushing = {
|
||||
req_id
|
||||
for req_id in done_pushing - failed_pushing
|
||||
if req_id in self._recving_metadata
|
||||
}
|
||||
# Remember failures until the final sibling WRITE completes.
|
||||
self._send_failures.update(failed_pushing)
|
||||
successful = {
|
||||
req_id
|
||||
for req_id in done_pushing - self._send_failures
|
||||
if req_id in self._reqs_to_send or req_id in self._reqs_to_process
|
||||
}
|
||||
self._send_failures.difference_update(done_pushing | done_sending)
|
||||
# Expired requests were already reported, even if their WRITEs finish later.
|
||||
done_pushing = successful
|
||||
for req_id in done_pushing:
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
self._reqs_to_process.discard(req_id)
|
||||
|
||||
@@ -116,9 +116,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
See https://platform.openai.com/docs/api-reference/completions/create
|
||||
for the API specification. This API mimics the OpenAI Completion API.
|
||||
|
||||
NOTE: Currently we do not support the following feature:
|
||||
- suffix (the language models we currently support do not support
|
||||
suffix)
|
||||
NOTE: suffix is only supported by models that implement FIM rendering.
|
||||
"""
|
||||
return await self._with_kv_transfer_rejection_cleanup(
|
||||
self._create_completion(request, raw_request), request, raw_request
|
||||
|
||||
@@ -31,11 +31,14 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
)
|
||||
|
||||
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
|
||||
if self.w_zp_name is not None and hasattr(layer, self.w_zp_name):
|
||||
name_map["zero_point"] = self.w_zp_name
|
||||
group_size = self.config.group_size
|
||||
quant_config = {
|
||||
"quant_method": "humming",
|
||||
"dtype": "int" + str(self.config.weight_type.size_bits),
|
||||
"group_size": 0 if group_size == -1 else group_size,
|
||||
"has_zero_point": self.config.zero_points,
|
||||
}
|
||||
|
||||
if self.config.zero_points:
|
||||
|
||||
@@ -337,6 +337,25 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
)
|
||||
intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K))
|
||||
|
||||
# Under EP, drop the top-k slots routed to remote experts at alignment
|
||||
# time instead of launching `off_experts == -1` GEMM blocks that only
|
||||
# write zeros; the pad-aware `moe_sum` below skips the same slots.
|
||||
# Their rows in the workspace caches are then never written, so every
|
||||
# consumer of those caches must be row-local: a dynamic per-tensor
|
||||
# scale for the second GEMM's activation would take its amax over the
|
||||
# untouched rows, and LoRA aligns the full `topk_ids` itself.
|
||||
a2_scale_is_global = (
|
||||
self.quant_dtype is not None
|
||||
and a2_scale is None
|
||||
and self.block_shape is None
|
||||
and (not self.per_act_token_quant or self.quantization_emulation)
|
||||
)
|
||||
skip_invalid = (
|
||||
expert_map is not None
|
||||
and self._lora_context is None
|
||||
and not a2_scale_is_global
|
||||
)
|
||||
|
||||
# Include fused shared-expert rows while preserving EP remapping.
|
||||
num_align_experts = w1.shape[0] if expert_map is None else global_num_experts
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = (
|
||||
@@ -350,6 +369,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
use_int8_w8a16=self.quant_config.use_int8_w8a16,
|
||||
use_int4_w4a16=self.quant_config.use_int4_w4a16,
|
||||
block_shape=self.block_shape,
|
||||
ignore_invalid_experts=skip_invalid,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -585,10 +605,21 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
)
|
||||
|
||||
# separate function is required for MoE + LoRA
|
||||
self.moe_sum(intermediate_cache3, output)
|
||||
self.moe_sum(intermediate_cache3, output, topk_ids, expert_map)
|
||||
|
||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
||||
ops.moe_sum(input, output)
|
||||
def moe_sum(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if expert_map is not None:
|
||||
# Skip the slots whose expert is not on this rank: the rows the
|
||||
# alignment dropped, or the zeros the `-1` blocks wrote.
|
||||
ops.moe_sum(input, output, topk_ids, expert_map)
|
||||
else:
|
||||
ops.moe_sum(input, output)
|
||||
|
||||
|
||||
class TritonWNA16Experts(TritonExperts):
|
||||
|
||||
@@ -41,6 +41,7 @@ if current_platform.is_cuda():
|
||||
def _dsv4_topk_kernel(
|
||||
gating_output_ptr,
|
||||
correction_bias_ptr,
|
||||
is_padding_ptr,
|
||||
topk_weights_ptr,
|
||||
topk_ids_ptr,
|
||||
routed_scaling_factor,
|
||||
@@ -49,6 +50,7 @@ if current_platform.is_cuda():
|
||||
image_sentinel_lo,
|
||||
NUM_EXPERTS: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
HAS_PADDING: tl.constexpr,
|
||||
HAS_VL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
@@ -106,6 +108,11 @@ if current_platform.is_cuda():
|
||||
output_mask = topk_offsets < 6
|
||||
output_offsets = row * 6 + topk_offsets
|
||||
|
||||
if HAS_PADDING:
|
||||
is_padding = tl.load(is_padding_ptr + row)
|
||||
selected_weights = tl.where(is_padding, 0.0, selected_weights)
|
||||
selected_ids = tl.where(is_padding, -1, selected_ids)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
@@ -118,11 +125,20 @@ def dsv4_topk(
|
||||
correction_bias: torch.Tensor,
|
||||
indices_dtype: torch.dtype,
|
||||
routed_scaling_factor: float,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens, num_experts = gating_output.shape
|
||||
if is_padding is not None:
|
||||
assert is_padding.dtype == torch.bool
|
||||
assert is_padding.shape == (num_tokens,)
|
||||
assert is_padding.device == gating_output.device
|
||||
assert is_padding.is_contiguous()
|
||||
assert indices_dtype in (torch.int32, torch.int64), (
|
||||
"Padding requires a signed indices dtype for the -1 sentinel."
|
||||
)
|
||||
has_vl = bias_vl is not None and image_sentinel_lo > 0
|
||||
if bias_vl is not None:
|
||||
assert input_ids is not None, "bias_vl routing requires input_ids"
|
||||
@@ -136,6 +152,7 @@ def dsv4_topk(
|
||||
_dsv4_topk_kernel[(num_tokens,)](
|
||||
gating_output,
|
||||
correction_bias,
|
||||
is_padding,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
routed_scaling_factor,
|
||||
@@ -144,6 +161,7 @@ def dsv4_topk(
|
||||
image_sentinel_lo,
|
||||
NUM_EXPERTS=num_experts,
|
||||
BLOCK_N=triton.next_power_of_2(num_experts),
|
||||
HAS_PADDING=is_padding is not None,
|
||||
HAS_VL=has_vl,
|
||||
num_warps=1,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
|
||||
@@ -148,9 +148,13 @@ def fused_topk_bias(
|
||||
)
|
||||
|
||||
output_indices_dtype = torch.int32 if indices_type is None else indices_type
|
||||
padding_mask = _get_padding_mask(gating_output.shape[0])
|
||||
if (
|
||||
scoring_func == "sqrtsoftplus"
|
||||
and hash_indices_table is None
|
||||
# dsv4_topk marks padded rows with a -1 sentinel, which unsigned
|
||||
# indices cannot represent; keep the generic path for that pair.
|
||||
and (padding_mask is None or output_indices_dtype != torch.uint32)
|
||||
and can_use_dsv4_topk(
|
||||
gating_output,
|
||||
e_score_correction_bias,
|
||||
@@ -165,6 +169,7 @@ def fused_topk_bias(
|
||||
e_score_correction_bias,
|
||||
output_indices_dtype,
|
||||
routed_scaling_factor,
|
||||
is_padding=padding_mask,
|
||||
input_ids=input_tokens,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
|
||||
@@ -787,6 +787,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
num_bits=weight_quant.num_bits,
|
||||
strategy=weight_quant.strategy,
|
||||
group_size=weight_quant.group_size,
|
||||
symmetric=weight_quant.symmetric,
|
||||
input_quant=input_quant,
|
||||
layer_name=layer_name,
|
||||
quant_format=format,
|
||||
@@ -799,6 +800,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
num_bits=weight_quant.num_bits,
|
||||
strategy=weight_quant.strategy,
|
||||
group_size=weight_quant.group_size,
|
||||
symmetric=weight_quant.symmetric,
|
||||
input_quant=input_quant,
|
||||
layer_name=layer_name,
|
||||
quant_format=format,
|
||||
|
||||
+9
-1
@@ -42,7 +42,15 @@ WNA16_SUPPORTED_TYPES_MAP = {
|
||||
7: scalar_types.uint7b64,
|
||||
8: scalar_types.uint8b128,
|
||||
}
|
||||
WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8}
|
||||
WNA16_ZP_SUPPORTED_TYPES_MAP = {
|
||||
2: scalar_types.uint2,
|
||||
3: scalar_types.uint3,
|
||||
4: scalar_types.uint4,
|
||||
5: scalar_types.uint5,
|
||||
6: scalar_types.uint6,
|
||||
7: scalar_types.uint7,
|
||||
8: scalar_types.uint8,
|
||||
}
|
||||
WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
|
||||
|
||||
|
||||
|
||||
+59
-14
@@ -1,11 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming.
|
||||
"""Weight N-bit INT scheme with INT4 activation quant via Humming.
|
||||
|
||||
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
|
||||
with INT4 symmetric dynamic per-token/per-group input activation
|
||||
quantization. Static, per-tensor, and asymmetric activation quantization
|
||||
are not supported.
|
||||
quantization. Supports both symmetric and asymmetric weight quantization.
|
||||
Static, per-tensor, and asymmetric activation quantization are not supported.
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
|
||||
WNA16_SUPPORTED_TYPES_MAP,
|
||||
WNA16_ZP_SUPPORTED_TYPES_MAP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_repeat_scales_on_all_ranks,
|
||||
@@ -36,6 +37,7 @@ from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
GroupQuantScaleParameter,
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
|
||||
@@ -52,6 +54,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
num_bits: int,
|
||||
strategy: str,
|
||||
group_size: int | None = None,
|
||||
symmetric: bool = True,
|
||||
input_quant: QuantizationArgs | None = None,
|
||||
layer_name: str | None = None,
|
||||
quant_format: str = "pack-quantized",
|
||||
@@ -60,6 +63,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
self.pack_factor = Fraction(32, num_bits)
|
||||
self.strategy = strategy
|
||||
self.group_size = -1 if group_size is None else group_size
|
||||
self.symmetric = symmetric
|
||||
self.input_quant = input_quant
|
||||
self.layer_name = layer_name
|
||||
self.quant_format = quant_format
|
||||
@@ -69,7 +73,19 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
f"Unsupported num_bits = {num_bits} for WNA4Int; "
|
||||
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
|
||||
)
|
||||
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
|
||||
|
||||
if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
|
||||
raise ValueError(
|
||||
f"Asymmetric quantization not supported for "
|
||||
f"num_bits = {num_bits}. Supported: "
|
||||
f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
|
||||
)
|
||||
|
||||
self.quant_type = (
|
||||
WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits]
|
||||
if not self.symmetric
|
||||
else WNA16_SUPPORTED_TYPES_MAP[num_bits]
|
||||
)
|
||||
|
||||
if input_quant is not None:
|
||||
if not input_quant.symmetric:
|
||||
@@ -143,7 +159,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
weight_type=self.quant_type,
|
||||
act_type=params_dtype,
|
||||
group_size=self.group_size,
|
||||
zero_points=False,
|
||||
zero_points=not self.symmetric,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_config)
|
||||
@@ -151,12 +167,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__)
|
||||
self._kernel_backends_being_used.add(kernel_type.__name__)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
)
|
||||
|
||||
input_quant_config = self._build_input_quant_config()
|
||||
if input_quant_config is not None:
|
||||
layer._humming_input_quant_config = input_quant_config
|
||||
@@ -167,10 +177,10 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
partition_scales = not marlin_repeat_scales_on_all_ranks(
|
||||
self.group_size, row_parallel
|
||||
)
|
||||
scales_size = input_size // group_size
|
||||
scales_and_zp_size = input_size // group_size
|
||||
if partition_scales:
|
||||
assert input_size_per_partition % group_size == 0
|
||||
scales_size = input_size_per_partition // group_size
|
||||
scales_and_zp_size = input_size_per_partition // group_size
|
||||
|
||||
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
|
||||
layer.register_parameter(
|
||||
@@ -190,7 +200,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
)
|
||||
|
||||
scale_data = torch.empty(
|
||||
output_size_per_partition, scales_size, dtype=params_dtype
|
||||
output_size_per_partition, scales_and_zp_size, dtype=params_dtype
|
||||
)
|
||||
if partition_scales:
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
@@ -207,6 +217,34 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
if not self.symmetric:
|
||||
packed_output_dim = math.ceil(
|
||||
output_size_per_partition * self.num_bits / 32
|
||||
)
|
||||
zeros_data = torch.zeros(
|
||||
packed_output_dim,
|
||||
scales_and_zp_size,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
if not partition_scales:
|
||||
qzeros = PackedColumnParameter(
|
||||
output_dim=0,
|
||||
packed_dim=0,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
data=zeros_data,
|
||||
)
|
||||
else:
|
||||
qzeros = PackedvLLMParameter(
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=0,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
data=zeros_data,
|
||||
)
|
||||
layer.register_parameter("weight_zero_point", qzeros)
|
||||
|
||||
layer.register_parameter(
|
||||
"weight_shape",
|
||||
BasevLLMParameter(
|
||||
@@ -215,6 +253,13 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
),
|
||||
)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
+59
-14
@@ -1,11 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming.
|
||||
"""Weight N-bit INT scheme with INT8 activation quant via Humming.
|
||||
|
||||
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
|
||||
with INT8 symmetric dynamic per-token/per-group input activation
|
||||
quantization. Static, per-tensor, and asymmetric activation quantization
|
||||
are not supported.
|
||||
quantization. Supports both symmetric and asymmetric weight quantization.
|
||||
Static, per-tensor, and asymmetric activation quantization are not supported.
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
|
||||
WNA16_SUPPORTED_TYPES_MAP,
|
||||
WNA16_ZP_SUPPORTED_TYPES_MAP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_repeat_scales_on_all_ranks,
|
||||
@@ -36,6 +37,7 @@ from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
GroupQuantScaleParameter,
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
|
||||
@@ -52,6 +54,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
num_bits: int,
|
||||
strategy: str,
|
||||
group_size: int | None = None,
|
||||
symmetric: bool = True,
|
||||
input_quant: QuantizationArgs | None = None,
|
||||
layer_name: str | None = None,
|
||||
quant_format: str = "pack-quantized",
|
||||
@@ -60,6 +63,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
self.pack_factor = Fraction(32, num_bits)
|
||||
self.strategy = strategy
|
||||
self.group_size = -1 if group_size is None else group_size
|
||||
self.symmetric = symmetric
|
||||
self.input_quant = input_quant
|
||||
self.layer_name = layer_name
|
||||
self.quant_format = quant_format
|
||||
@@ -69,7 +73,19 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
f"Unsupported num_bits = {num_bits} for WNA8Int; "
|
||||
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
|
||||
)
|
||||
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
|
||||
|
||||
if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
|
||||
raise ValueError(
|
||||
f"Asymmetric quantization not supported for "
|
||||
f"num_bits = {num_bits}. Supported: "
|
||||
f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
|
||||
)
|
||||
|
||||
self.quant_type = (
|
||||
WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits]
|
||||
if not self.symmetric
|
||||
else WNA16_SUPPORTED_TYPES_MAP[num_bits]
|
||||
)
|
||||
|
||||
if input_quant is not None:
|
||||
if not input_quant.symmetric:
|
||||
@@ -143,7 +159,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
weight_type=self.quant_type,
|
||||
act_type=params_dtype,
|
||||
group_size=self.group_size,
|
||||
zero_points=False,
|
||||
zero_points=not self.symmetric,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_config)
|
||||
@@ -151,12 +167,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__)
|
||||
self._kernel_backends_being_used.add(kernel_type.__name__)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
)
|
||||
|
||||
input_quant_config = self._build_input_quant_config()
|
||||
if input_quant_config is not None:
|
||||
layer._humming_input_quant_config = input_quant_config
|
||||
@@ -167,10 +177,10 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
partition_scales = not marlin_repeat_scales_on_all_ranks(
|
||||
self.group_size, row_parallel
|
||||
)
|
||||
scales_size = input_size // group_size
|
||||
scales_and_zp_size = input_size // group_size
|
||||
if partition_scales:
|
||||
assert input_size_per_partition % group_size == 0
|
||||
scales_size = input_size_per_partition // group_size
|
||||
scales_and_zp_size = input_size_per_partition // group_size
|
||||
|
||||
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
|
||||
layer.register_parameter(
|
||||
@@ -190,7 +200,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
)
|
||||
|
||||
scale_data = torch.empty(
|
||||
output_size_per_partition, scales_size, dtype=params_dtype
|
||||
output_size_per_partition, scales_and_zp_size, dtype=params_dtype
|
||||
)
|
||||
if partition_scales:
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
@@ -207,6 +217,34 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
if not self.symmetric:
|
||||
packed_output_dim = math.ceil(
|
||||
output_size_per_partition * self.num_bits / 32
|
||||
)
|
||||
zeros_data = torch.zeros(
|
||||
packed_output_dim,
|
||||
scales_and_zp_size,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
if not partition_scales:
|
||||
qzeros = PackedColumnParameter(
|
||||
output_dim=0,
|
||||
packed_dim=0,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
data=zeros_data,
|
||||
)
|
||||
else:
|
||||
qzeros = PackedvLLMParameter(
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=0,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
data=zeros_data,
|
||||
)
|
||||
layer.register_parameter("weight_zero_point", qzeros)
|
||||
|
||||
layer.register_parameter(
|
||||
"weight_shape",
|
||||
BasevLLMParameter(
|
||||
@@ -215,6 +253,13 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
),
|
||||
)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import fnmatch
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, cast
|
||||
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -82,10 +81,6 @@ __all__ = ["QuarkLinearMethod"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# model_type values that use dynamic MXFP4 re-quantization for
|
||||
# OCP MX fp4 Quark checkpoints
|
||||
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"})
|
||||
|
||||
|
||||
class QuantKeyMatch(NamedTuple):
|
||||
matches: bool
|
||||
@@ -119,37 +114,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
self.kv_cache_group = kv_cache_group
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.pack_method = pack_method
|
||||
# Note : this flag is kept disabled because the overhead of
|
||||
# dynamic mxfp4 quantization negates the performance gains
|
||||
# that come from shifting to mxfp4. It is left here in case
|
||||
# we want to re-enable it in the future.
|
||||
self.dynamic_mxfp4_quant = False
|
||||
|
||||
def maybe_update_config(
|
||||
self,
|
||||
model_name: str,
|
||||
hf_config: PretrainedConfig | None = None,
|
||||
revision: str | None = None,
|
||||
):
|
||||
"""Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints."""
|
||||
if hf_config is None:
|
||||
return
|
||||
|
||||
if (
|
||||
getattr(hf_config, "model_type", None)
|
||||
not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
|
||||
):
|
||||
return
|
||||
|
||||
quant_config = getattr(hf_config, "quantization_config", None)
|
||||
if isinstance(quant_config, dict):
|
||||
quant_dtype = (
|
||||
quant_config.get("global_quant_config", {})
|
||||
.get("weight", {})
|
||||
.get("dtype")
|
||||
)
|
||||
if quant_dtype == "fp4":
|
||||
self.dynamic_mxfp4_quant = True
|
||||
|
||||
def get_linear_method(self) -> "QuarkLinearMethod":
|
||||
return QuarkLinearMethod(self)
|
||||
@@ -201,17 +165,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
self.get_quant_method_target(prefix, type(layer))
|
||||
)
|
||||
|
||||
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
|
||||
is_ignored = should_ignore_layer(
|
||||
prefix,
|
||||
ignore=exclude_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
check_children=isinstance(layer, RoutedExperts),
|
||||
)
|
||||
dynamic_mxfp4_quant = (
|
||||
is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant
|
||||
)
|
||||
|
||||
if method_cls is UnquantizedFusedMoEMethod:
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if method_cls is UnquantizedLinearMethod:
|
||||
@@ -222,7 +175,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
scheme_cls,
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
|
||||
weight_config=self._find_matched_config(prefix, type(layer)).get(
|
||||
"weight"
|
||||
),
|
||||
@@ -274,11 +226,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
if is_routed_experts:
|
||||
return None, None, UnquantizedFusedMoEMethod
|
||||
if issubclass(layer_type, LinearBase):
|
||||
if "self_attn" in prefix and self.dynamic_mxfp4_quant:
|
||||
weight_quant_key, activation_key, _ = self.get_scheme_cls(
|
||||
layer_type, prefix
|
||||
)
|
||||
return weight_quant_key, activation_key, QuarkLinearMethod
|
||||
return None, None, UnquantizedLinearMethod
|
||||
return None, None, None
|
||||
if issubclass(layer_type, LinearBase):
|
||||
@@ -956,7 +903,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
scheme_cls: type["QuarkScheme"],
|
||||
weight_quant_key: QuantKey | None,
|
||||
activation_quant_key: QuantKey | None,
|
||||
dynamic_mxfp4_quant: bool = False,
|
||||
weight_config: dict[str, Any] | None = None,
|
||||
) -> "QuarkScheme":
|
||||
"""Construct a Quark scheme selected by get_scheme_cls."""
|
||||
@@ -996,9 +942,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
):
|
||||
kwargs["weight_quant_key"] = weight_quant_key
|
||||
|
||||
if scheme_cls is QuarkOCP_MX:
|
||||
kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant
|
||||
|
||||
scheme = scheme_cls(**kwargs)
|
||||
|
||||
# Raise error if device does not support the scheme
|
||||
|
||||
@@ -26,10 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .quark_scheme import QuarkScheme
|
||||
@@ -46,10 +44,9 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
dynamic_mxfp4_quant: bool = False,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
|
||||
|
||||
self.weight_dtype = next(
|
||||
dtype
|
||||
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
|
||||
@@ -109,21 +106,9 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
def get_min_capability(cls) -> int:
|
||||
return 70
|
||||
|
||||
def process_dynamic_mxfp4_weights_after_loading(
|
||||
self, layer: torch.nn.Module
|
||||
) -> None:
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
w_q, w_s = dynamic_mxfp4_quant(layer.weight)
|
||||
layer.weight_scale = torch.nn.Parameter(w_s, requires_grad=False)
|
||||
layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
|
||||
|
||||
if self.dynamic_mxfp4_quant:
|
||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||
|
||||
self.ocp_mx_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
@@ -145,51 +130,36 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
"tensor-parallel sharding for this layer."
|
||||
)
|
||||
|
||||
if self.dynamic_mxfp4_quant:
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, kwargs)
|
||||
else:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
# WEIGHT
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=1,
|
||||
packed_factor=self.packed_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=1,
|
||||
packed_factor=self.packed_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT SCALE
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
# WEIGHT SCALE
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
if self.weight_quant_key == kMxfp4Static:
|
||||
self.ocp_mx_linear = init_mxfp4_linear_kernel(
|
||||
|
||||
@@ -101,6 +101,19 @@ def prepare_humming_linear_layer_config(
|
||||
param_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
if (
|
||||
weight_schema.has_zero_point
|
||||
and input_schema.a_dtype is not None
|
||||
and weight_schema.b_dtype.num_bits == input_schema.a_dtype.num_bits
|
||||
):
|
||||
raise ValueError(
|
||||
"Humming does not support asymmetric (zero_point) weights "
|
||||
"when weight and activation bit widths are equal "
|
||||
f"(W{weight_schema.b_dtype.num_bits}"
|
||||
f"A{input_schema.a_dtype.num_bits}). "
|
||||
"Use symmetric weights or a different activation bit width."
|
||||
)
|
||||
|
||||
tensors.update(input_tensors)
|
||||
layer.weight_schema = weight_schema
|
||||
input_schema = check_and_fallback_input_schema(
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from .base import RotaryEmbeddingBase
|
||||
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding, yarn_get_mscale
|
||||
@@ -240,6 +241,62 @@ def apply_interleaved_rope(x: torch.Tensor, mrope_section: list[int]) -> torch.T
|
||||
return torch.where(is_width, x[2], result)
|
||||
|
||||
|
||||
def _mrope_apply(
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
mrope_section_t: int,
|
||||
mrope_section_h: int,
|
||||
mrope_section_w: int,
|
||||
mrope_interleaved: bool,
|
||||
is_neox_style: bool,
|
||||
) -> None:
|
||||
cos_sin = cos_sin_cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
query_out, key_out = triton_mrope(
|
||||
query,
|
||||
key,
|
||||
cos,
|
||||
sin,
|
||||
[mrope_section_t, mrope_section_h, mrope_section_w],
|
||||
head_size,
|
||||
rotary_dim,
|
||||
mrope_interleaved,
|
||||
is_neox_style,
|
||||
)
|
||||
if query_out.data_ptr() != query.data_ptr():
|
||||
query.copy_(query_out.view_as(query))
|
||||
if key_out.data_ptr() != key.data_ptr():
|
||||
key.copy_(key_out.view_as(key))
|
||||
|
||||
|
||||
def _mrope_apply_fake(
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
mrope_section_t: int,
|
||||
mrope_section_h: int,
|
||||
mrope_section_w: int,
|
||||
mrope_interleaved: bool,
|
||||
is_neox_style: bool,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mrope",
|
||||
op_func=_mrope_apply,
|
||||
mutates_args=["query", "key"],
|
||||
fake_impl=_mrope_apply_fake,
|
||||
)
|
||||
|
||||
|
||||
class MRotaryEmbedding(RotaryEmbeddingBase):
|
||||
"""Rotary Embedding with Multimodal Sections."""
|
||||
|
||||
@@ -426,6 +483,35 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
|
||||
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
|
||||
return query, key
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if positions.ndim == 2:
|
||||
assert key is not None
|
||||
assert self.mrope_section
|
||||
query_shape = query.shape
|
||||
key_shape = key.shape
|
||||
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
|
||||
torch.ops.vllm.mrope(
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
cos_sin_cache,
|
||||
self.head_size,
|
||||
self.rotary_dim,
|
||||
self.mrope_section[0],
|
||||
self.mrope_section[1],
|
||||
self.mrope_section[2],
|
||||
self.mrope_interleaved,
|
||||
self.is_neox_style,
|
||||
)
|
||||
return query.reshape(query_shape), key.reshape(key_shape)
|
||||
return super().forward_hip(positions, query, key)
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
|
||||
@@ -498,11 +498,11 @@ class CpuPlatform(Platform):
|
||||
return
|
||||
|
||||
# reconcile attention and mamba page sizes
|
||||
backend_cls = cls._find_non_ssm_backend(vllm_config)
|
||||
if backend_cls is None:
|
||||
backend_classes = cls._find_non_ssm_backends(vllm_config)
|
||||
if not backend_classes:
|
||||
return
|
||||
|
||||
cls._align_hybrid_block_size(vllm_config, backend_cls)
|
||||
cls._align_hybrid_block_size(vllm_config, backend_classes[0])
|
||||
|
||||
@classmethod
|
||||
def discover_numa_topology(cls) -> list[list[int]]:
|
||||
|
||||
+72
-18
@@ -3,6 +3,8 @@
|
||||
import contextlib
|
||||
import enum
|
||||
import functools
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
@@ -573,10 +575,10 @@ class Platform:
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def _find_non_ssm_backend(
|
||||
def _find_non_ssm_backends(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
) -> "type[AttentionBackend] | None":
|
||||
"""Find the first non-SSM attention backend from model layers."""
|
||||
) -> "list[type[AttentionBackend]]":
|
||||
"""Distinct non-SSM attention backends, in layer order."""
|
||||
from vllm.config.vllm import get_layers_from_vllm_config
|
||||
from vllm.model_executor.layers.attention_layer_base import (
|
||||
AttentionLayerBase,
|
||||
@@ -586,11 +588,62 @@ class Platform:
|
||||
vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
)
|
||||
backends: list[type[AttentionBackend]] = []
|
||||
for layer in attn_layers.values():
|
||||
b = layer.get_attn_backend()
|
||||
if not b.is_ssm():
|
||||
return b
|
||||
return None
|
||||
if not b.is_ssm() and b not in backends:
|
||||
backends.append(b)
|
||||
return backends
|
||||
|
||||
@classmethod
|
||||
def _preferred_block_size_for_backends(
|
||||
cls,
|
||||
backend_classes: "list[type[AttentionBackend]]",
|
||||
default_block_size: int,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> int:
|
||||
"""Smallest block size every backend accepts.
|
||||
|
||||
``supports_block_size`` may be overridden to accept exact sizes only
|
||||
(CPU_MLA takes 16 and no multiple of it), so candidates are the LCMs of
|
||||
one supported size per backend, each checked against every backend.
|
||||
|
||||
Raises:
|
||||
ValueError: If the backends share no supported block size.
|
||||
|
||||
"""
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
# Backends may read the current config to decide a preference.
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if len(backend_classes) == 1:
|
||||
return backend_classes[0].get_preferred_block_size(default_block_size)
|
||||
if all(b.supports_block_size(default_block_size) for b in backend_classes):
|
||||
return default_block_size
|
||||
# A backend declaring no sizes accepts any, so it contributes 1.
|
||||
per_backend_sizes = [
|
||||
[
|
||||
s.base if isinstance(s, MultipleOf) else s
|
||||
for s in b.get_supported_kernel_block_sizes()
|
||||
]
|
||||
or [1]
|
||||
for b in backend_classes
|
||||
]
|
||||
candidates = sorted(
|
||||
{math.lcm(*sizes) for sizes in itertools.product(*per_backend_sizes)}
|
||||
)
|
||||
for candidate in candidates:
|
||||
if all(b.supports_block_size(candidate) for b in backend_classes):
|
||||
return candidate
|
||||
raise ValueError(
|
||||
"The attention backends share no supported KV cache block size ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
|
||||
for b in backend_classes
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
@@ -598,7 +651,6 @@ class Platform:
|
||||
For hybrid models, also aligns block_size with mamba page sizes.
|
||||
"""
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
@@ -607,34 +659,36 @@ class Platform:
|
||||
if not model_config:
|
||||
return
|
||||
|
||||
backend_cls = cls._find_non_ssm_backend(vllm_config)
|
||||
if backend_cls is None:
|
||||
backend_classes = cls._find_non_ssm_backends(vllm_config)
|
||||
if not backend_classes:
|
||||
return
|
||||
|
||||
# Phase 1: Pick block size from backend (skip if user set --block-size)
|
||||
# Phase 1: Pick a block size every attention backend supports (skip if
|
||||
# user set --block-size). Models can mix backends with disjoint
|
||||
# preferences, and a size from the first alone later fails
|
||||
# select_common_block_size().
|
||||
if not cache_config.user_specified_block_size:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
preferred = backend_cls.get_preferred_block_size(
|
||||
CacheConfig.DEFAULT_BLOCK_SIZE
|
||||
)
|
||||
preferred = cls._preferred_block_size_for_backends(
|
||||
backend_classes, CacheConfig.DEFAULT_BLOCK_SIZE, vllm_config
|
||||
)
|
||||
if preferred != CacheConfig.DEFAULT_BLOCK_SIZE:
|
||||
logger.info(
|
||||
"Setting kv cache block size to %d for %s backend.",
|
||||
"Setting kv cache block size to %d for %s backend(s).",
|
||||
preferred,
|
||||
backend_cls.get_name(),
|
||||
"/".join(b.get_name() for b in backend_classes),
|
||||
)
|
||||
cache_config.block_size = preferred
|
||||
|
||||
# Phase 2: Align block/mamba sizes for hybrid models
|
||||
# (may override user settings).
|
||||
if model_config.is_hybrid:
|
||||
cls._align_hybrid_block_size(vllm_config, backend_cls)
|
||||
cls._align_hybrid_block_size(vllm_config, backend_classes[0])
|
||||
|
||||
# Phase 3: Align block/page sizes when multiple KV dtypes share the
|
||||
# block pool (e.g. nvfp4 primary + unquantized skip layers).
|
||||
# May override the user's --block-size.
|
||||
if cache_config.kv_cache_dtype_skip_layers:
|
||||
cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls)
|
||||
cls._align_heterogeneous_kv_block_size(vllm_config, backend_classes[0])
|
||||
|
||||
@classmethod
|
||||
def _align_heterogeneous_kv_block_size(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.metadata
|
||||
import os
|
||||
import platform
|
||||
from datetime import timedelta
|
||||
@@ -361,6 +362,25 @@ def get_cdna_version() -> int:
|
||||
return 0
|
||||
|
||||
|
||||
@cache
|
||||
def get_rocm_version() -> tuple[int, ...] | None:
|
||||
"""Return the installed ROCm release as (major, minor, patch), or None."""
|
||||
# ROCm 10+ ships as the `rocm` pip SDK; older releases install to /opt/rocm.
|
||||
try:
|
||||
version = importlib.metadata.version("rocm")
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
rocm_path = os.environ.get("ROCM_PATH", "/opt/rocm")
|
||||
try:
|
||||
with open(os.path.join(rocm_path, ".info", "version")) as f:
|
||||
version = f.read()
|
||||
except OSError:
|
||||
return None
|
||||
match = re.match(r"\s*(\d+)\.(\d+)(?:\.(\d+))?", version)
|
||||
if match is None:
|
||||
return None
|
||||
return tuple(int(part) for part in match.groups() if part is not None)
|
||||
|
||||
|
||||
# Enable HIP online tuning early, before hipBLASLt initializes.
|
||||
# Turn on hipBLASLt online tuning if use AITER hipBLASLt GEMM.
|
||||
if (
|
||||
|
||||
@@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
return tokenizer
|
||||
|
||||
def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
|
||||
"""Render OpenAI completion suffix input when the renderer supports FIM."""
|
||||
return None
|
||||
|
||||
def _decode(self, *args, **kwargs):
|
||||
return self.get_tokenizer().decode(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -16,6 +16,10 @@ from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
|
||||
_FIM_BEGIN = "<|fim▁begin|>"
|
||||
_FIM_HOLE = "<|fim▁hole|>"
|
||||
_FIM_END = "<|fim▁end|>"
|
||||
|
||||
|
||||
class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
|
||||
def __init__(
|
||||
@@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
|
||||
def _apply_chat_template(self, *args, **kwargs):
|
||||
return self.get_tokenizer().apply_chat_template(*args, **kwargs)
|
||||
|
||||
def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
|
||||
return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}"
|
||||
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
|
||||
@@ -4,7 +4,7 @@ import time
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias, cast
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
|
||||
from openai.types.responses.tool import Mcp, Tool
|
||||
@@ -548,9 +548,58 @@ class OnlineRenderer:
|
||||
Called directly by render_completion_request and delegated to by
|
||||
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
|
||||
"""
|
||||
# Return error for unsupported features.
|
||||
prompt_input = request.prompt
|
||||
if request.suffix is not None:
|
||||
return self.create_error_response("suffix is not currently supported")
|
||||
if request.echo:
|
||||
return self.create_error_response(
|
||||
"Echo is unsupported with suffix.",
|
||||
param="suffix",
|
||||
)
|
||||
|
||||
if request.prompt_embeds is not None:
|
||||
return self.create_error_response(
|
||||
"suffix is not supported with prompt_embeds",
|
||||
param="suffix",
|
||||
)
|
||||
|
||||
if request.truncate_prompt_tokens is not None:
|
||||
return self.create_error_response(
|
||||
"suffix is not supported with truncate_prompt_tokens",
|
||||
param="suffix",
|
||||
)
|
||||
|
||||
if isinstance(request.prompt, str):
|
||||
rendered_prompt = self.renderer.render_completion_suffix(
|
||||
request.prompt, request.suffix
|
||||
)
|
||||
if rendered_prompt is None:
|
||||
return self.create_error_response(
|
||||
"suffix is only supported for models with FIM completion "
|
||||
"rendering",
|
||||
param="suffix",
|
||||
)
|
||||
prompt_input = rendered_prompt
|
||||
elif isinstance(request.prompt, list) and all(
|
||||
isinstance(prompt, str) for prompt in request.prompt
|
||||
):
|
||||
rendered_prompts = []
|
||||
for prompt in cast(list[str], request.prompt):
|
||||
rendered_prompt = self.renderer.render_completion_suffix(
|
||||
prompt, request.suffix
|
||||
)
|
||||
if rendered_prompt is None:
|
||||
return self.create_error_response(
|
||||
"suffix is only supported for models with FIM completion "
|
||||
"rendering",
|
||||
param="suffix",
|
||||
)
|
||||
rendered_prompts.append(rendered_prompt)
|
||||
prompt_input = rendered_prompts
|
||||
else:
|
||||
return self.create_error_response(
|
||||
"suffix requires text prompt input for FIM completion rendering",
|
||||
param="suffix",
|
||||
)
|
||||
|
||||
if request.echo and request.prompt_embeds is not None:
|
||||
return self.create_error_response("Echo is unsupported with prompt embeds.")
|
||||
@@ -562,7 +611,7 @@ class OnlineRenderer:
|
||||
|
||||
engine_inputs = await self.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.prompt,
|
||||
prompt_input=prompt_input,
|
||||
prompt_embeds=request.prompt_embeds,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
@@ -316,8 +316,13 @@ class ScalarType:
|
||||
|
||||
|
||||
class scalar_types:
|
||||
uint2 = ScalarType.uint(2, None)
|
||||
uint3 = ScalarType.uint(3, None)
|
||||
int4 = ScalarType.int_(4, None)
|
||||
uint4 = ScalarType.uint(4, None)
|
||||
uint5 = ScalarType.uint(5, None)
|
||||
uint6 = ScalarType.uint(6, None)
|
||||
uint7 = ScalarType.uint(7, None)
|
||||
int8 = ScalarType.int_(8, None)
|
||||
uint8 = ScalarType.uint(8, None)
|
||||
float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN)
|
||||
|
||||
@@ -110,7 +110,7 @@ class OvisProcessor(ProcessorMixin):
|
||||
) -> BatchFeature:
|
||||
"""Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
|
||||
and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode
|
||||
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to
|
||||
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwargs` arguments to
|
||||
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -115,7 +115,7 @@ class Ovis2_5Processor(ProcessorMixin):
|
||||
and image(s). This method forwards the `text`and `kwargs` arguments
|
||||
to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text`
|
||||
is not `None` to encode the text. To prepare the vision inputs,
|
||||
this method forwards the `vision_infos` and `kwrags` arguments to
|
||||
this method forwards the `vision_infos` and `kwargs` arguments to
|
||||
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`]
|
||||
if `vision_infos` is not `None`.
|
||||
|
||||
|
||||
@@ -926,6 +926,10 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
|
||||
"""
|
||||
return False
|
||||
|
||||
def fused_qk_norm_mrope_kvcache_supported(self):
|
||||
"""Whether this implementation supports fused QKNorm+MRoPE+KVCache."""
|
||||
return False
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
"""Does this attention implementation support RoPE+KVCache fusion.
|
||||
This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops
|
||||
@@ -955,6 +959,26 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def do_qk_norm_mrope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
qkv: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
mrope_section: tuple[int, int, int],
|
||||
is_interleaved: bool,
|
||||
rotary_dim: int,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
"""Apply QK-norm and MRoPE, then write K/V to the cache."""
|
||||
raise NotImplementedError
|
||||
|
||||
def do_rope_and_kv_cache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
|
||||
@@ -406,6 +406,15 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
# The CPU kernel executes attention sinks natively in bf16. If the
|
||||
# sinks tensor is anything other than bf16, cast it to fp32 so it is
|
||||
# executed in full float precision (done lazily here, after weights
|
||||
# are loaded, rather than at __init__ time).
|
||||
if self.sinks is not None and self.sinks.dtype not in [
|
||||
torch.bfloat16,
|
||||
torch.float32,
|
||||
]:
|
||||
self.sinks = self.sinks.to(torch.float32)
|
||||
ops.cpu_attention_with_kv_cache(
|
||||
query=query[:num_actual_tokens],
|
||||
key_cache=key_cache,
|
||||
|
||||
@@ -772,9 +772,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
device=device,
|
||||
)
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
max_num_partial_tiles = reduce_partial_map_size
|
||||
# AITER metadata sizing assumes all requests can carry max_prefill_qlen tokens,
|
||||
# which is a loose worst case in the number of QO tiles. Rather, the sum of
|
||||
# qlens is bounded by max_num_batched_tokens. Manually compute the number of
|
||||
# qo tiles to avoid OOM at startup.
|
||||
# TODO: AITER should give us this budget constrained value
|
||||
qo_tile_cnt = (
|
||||
cdiv(max_num_batched_tokens, _FP8_PREFILL_TILE_Q) + max_num_reqs - 1
|
||||
)
|
||||
max_num_partial_tiles = qo_tile_cnt + current_platform.num_compute_units()
|
||||
reservations: list[tuple[tuple[int, ...], torch.dtype]] = [
|
||||
(
|
||||
(max_num_partial_tiles * _FP8_PREFILL_TILE_Q, num_head_k, v_head_dim),
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import ClassVar
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
@@ -359,6 +359,9 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def fused_qk_norm_mrope_kvcache_supported(self) -> bool:
|
||||
return IS_AITER_FOUND
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
@@ -397,6 +400,47 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
use_shuffle_layout=False,
|
||||
)
|
||||
|
||||
def do_qk_norm_mrope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
qkv: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
mrope_section: tuple[int, int, int],
|
||||
is_interleaved: bool,
|
||||
rotary_dim: int,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||
rocm_aiter_ops.do_qk_norm_mrope_kvcache_update(
|
||||
qkv=qkv,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
num_heads_q=self.num_heads,
|
||||
num_heads_k=self.num_kv_heads,
|
||||
head_dim=self.head_size,
|
||||
is_neox=is_neox,
|
||||
mrope_section=list(mrope_section),
|
||||
is_interleaved=is_interleaved,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
q_out=q_out,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
slot_mapping=layer_slot_mapping,
|
||||
k_scale=layer._k_scale_cpu,
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
rotary_dim=rotary_dim,
|
||||
)
|
||||
|
||||
def do_rope_and_kv_cache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
|
||||
@@ -437,7 +437,7 @@ class KVCacheManager:
|
||||
----------------------------------------------------------------------
|
||||
```
|
||||
|
||||
Abbrivations:
|
||||
Abbreviations:
|
||||
|
||||
```
|
||||
comp = request.num_computed_tokens
|
||||
|
||||
@@ -555,6 +555,11 @@ def _post_update_kernel(
|
||||
all_token_ids_ptr,
|
||||
all_token_ids_stride,
|
||||
total_len_ptr,
|
||||
broadcast_drafts_ptr,
|
||||
broadcast_drafts_stride,
|
||||
draft_tokens_ptr,
|
||||
draft_tokens_stride,
|
||||
num_spec,
|
||||
):
|
||||
req_id = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_id)
|
||||
@@ -562,6 +567,17 @@ def _post_update_kernel(
|
||||
# Filter rows with negative index entries.
|
||||
return
|
||||
|
||||
if broadcast_drafts_ptr is not None:
|
||||
# PP path: adopt the draft tokens proposed by the last rank's
|
||||
# speculator so the next verification step embeds the real drafts.
|
||||
for i in range(num_spec):
|
||||
token_id = tl.load(
|
||||
broadcast_drafts_ptr + req_id * broadcast_drafts_stride + i
|
||||
)
|
||||
tl.store(
|
||||
draft_tokens_ptr + req_state_idx * draft_tokens_stride + i, token_id
|
||||
)
|
||||
|
||||
total_len = tl.load(total_len_ptr + req_state_idx)
|
||||
num_sampled = tl.load(num_sampled_ptr + req_id)
|
||||
if num_sampled > 0:
|
||||
@@ -622,6 +638,11 @@ def post_update(
|
||||
all_token_ids: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
total_len: torch.Tensor,
|
||||
# [num_reqs, num_spec]; drafts broadcast from the last PP rank. Only
|
||||
# passed on non-last PP ranks, which never run the speculator.
|
||||
broadcast_drafts: torch.Tensor | None = None,
|
||||
# [max_num_reqs, num_spec]
|
||||
draft_tokens_out: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
_post_update_kernel[(num_reqs,)](
|
||||
@@ -638,6 +659,11 @@ def post_update(
|
||||
all_token_ids,
|
||||
all_token_ids.stride(0),
|
||||
total_len,
|
||||
broadcast_drafts,
|
||||
broadcast_drafts.stride(0) if broadcast_drafts is not None else 0,
|
||||
draft_tokens_out,
|
||||
draft_tokens_out.stride(0) if draft_tokens_out is not None else 0,
|
||||
broadcast_drafts.shape[1] if broadcast_drafts is not None else 0,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
@@ -860,6 +860,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# dummy run the eagle speculator's propose to ensure DP/EP sync.
|
||||
if self.speculator is not None:
|
||||
assert self.sampler is not None
|
||||
assert hidden_states is not None
|
||||
self.step_timing.drafter_start()
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
|
||||
if self.speculator.supports_mm_inputs:
|
||||
@@ -875,7 +876,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
spec_hidden_states = hidden_states
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
|
||||
if pre_hc_hidden_states is not None:
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]]
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler._get_contexts(input_batch.idx_mapping),
|
||||
@@ -1107,6 +1109,44 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if outputs is not None:
|
||||
self.postprocess_sampled(**outputs)
|
||||
|
||||
def warmup_pp_decode_update(self) -> None:
|
||||
"""JIT-compile the kernel behind ``update_pp_decode_requests``.
|
||||
|
||||
That path only runs on real steps, so the warmup steps never reach it
|
||||
on non-last PP ranks. Its first triton compile must not happen
|
||||
mid-serving: the in-flight sampled-token broadcast keeps a NCCL kernel
|
||||
spinning on this device, which blocks the CUDA module load and
|
||||
deadlocks the pipeline. An all -1 idx_mapping makes this a no-op.
|
||||
The freshly allocated int32 tensors are 16-byte aligned, matching the
|
||||
padded views `PPHandler` produces at serving time (triton specializes
|
||||
on pointer alignment).
|
||||
"""
|
||||
assert self.pp_handler is not None
|
||||
num_spec = self.pp_handler.max_sample_len - 1
|
||||
broadcast_drafts = (
|
||||
torch.zeros((1, num_spec), dtype=torch.int64, device=self.device)
|
||||
if num_spec > 0
|
||||
else None
|
||||
)
|
||||
post_update(
|
||||
torch.full((1,), -1, dtype=torch.int64, device=self.device),
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
self.req_states.last_sampled_tokens,
|
||||
None,
|
||||
torch.zeros(
|
||||
(1, self.pp_handler.max_sample_len),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
),
|
||||
torch.zeros(1, dtype=torch.int32, device=self.device),
|
||||
torch.zeros(1, dtype=torch.int32, device=self.device),
|
||||
None,
|
||||
self.req_states.all_token_ids.gpu,
|
||||
self.req_states.total_len.gpu,
|
||||
broadcast_drafts,
|
||||
self.req_states.draft_tokens if broadcast_drafts is not None else None,
|
||||
)
|
||||
|
||||
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
for new_req_data in scheduler_output.scheduled_new_reqs:
|
||||
assert new_req_data.prefill_token_ids is not None
|
||||
@@ -1588,6 +1628,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_sampled: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
broadcast_drafts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# Update the number of computed tokens.
|
||||
output_bin_counts = None
|
||||
@@ -1605,6 +1646,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
query_start_loc,
|
||||
self.req_states.all_token_ids.gpu,
|
||||
self.req_states.total_len.gpu,
|
||||
broadcast_drafts,
|
||||
self.req_states.draft_tokens if broadcast_drafts is not None else None,
|
||||
)
|
||||
|
||||
self.model_state.postprocess_state(
|
||||
@@ -2125,7 +2168,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
spec_hidden_states = draft_hidden_states
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)]
|
||||
if pre_hc_hidden_states is not None:
|
||||
spec_hidden_states = pre_hc_hidden_states[
|
||||
: draft_hidden_states.size(0)
|
||||
]
|
||||
if isinstance(self.sampler, GPUWatermarkSampler):
|
||||
self.speculator.prepare_watermarking(
|
||||
self.sampler._get_contexts(input_batch.idx_mapping),
|
||||
@@ -2147,8 +2193,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dp_sync=dp_sync,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||
if self.adaptive_verification is not None:
|
||||
if draft_tokens is not None:
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||
if self.pp_handler is not None:
|
||||
# Earlier stages never run the speculator; ship the
|
||||
# drafts so their next verification step embeds the real
|
||||
# draft tokens instead of stale buffer contents.
|
||||
self.pp_handler.broadcast_drafts(
|
||||
self.req_states.draft_tokens, input_batch
|
||||
)
|
||||
if draft_tokens is not None and self.adaptive_verification is not None:
|
||||
self.adaptive_verification.record_confidences(
|
||||
self.speculator.draft_token_confidence_probs, input_batch
|
||||
)
|
||||
@@ -2160,7 +2214,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch,
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping],
|
||||
)
|
||||
if self.pp_handler is not None:
|
||||
if self.pp_handler is not None and self.speculator is None:
|
||||
# When a speculator ran, the propose() path above already
|
||||
# broadcast the fresh drafts. Broadcasting here as well would
|
||||
# double-post on the pp_broadcast group and misalign the
|
||||
# recv FIFO on earlier stages, hanging the pipeline.
|
||||
self.pp_handler.broadcast_drafts(
|
||||
self.req_states.draft_tokens, input_batch
|
||||
)
|
||||
|
||||
@@ -46,6 +46,18 @@ def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
return produces_sample if produces_sample.any() else None
|
||||
|
||||
|
||||
def _alloc_combined(num_reqs: int, device: torch.device) -> torch.Tensor:
|
||||
"""Allocate the (2, N) int32 buffer broadcast alongside sampled tokens.
|
||||
|
||||
The inner dim is padded to a multiple of 4 so that both `unbind` views
|
||||
stay 16-byte aligned for any `num_reqs`: triton specializes on pointer
|
||||
alignment, and a misaligned `num_rejected` would JIT-compile a second
|
||||
`_post_update_kernel` variant at serving time. The padding is broadcast
|
||||
but never read. Sender and receiver must both use this allocation.
|
||||
"""
|
||||
return torch.empty(2, -(-num_reqs // 4) * 4, dtype=torch.int32, device=device)
|
||||
|
||||
|
||||
class PPHandler:
|
||||
"""Runs the PP sampled-token broadcast/recv on a side stream so the
|
||||
default stream isn't gated by the matching peer call. Step T's recv is
|
||||
@@ -86,6 +98,14 @@ class PPHandler:
|
||||
)
|
||||
self.aux_hidden_state_relay_keys: tuple[str, ...] = ()
|
||||
|
||||
# Warmup steps run the pipeline with synthetic batches whose outputs are
|
||||
# discarded; the sampled-token broadcast is disabled there so its
|
||||
# side-stream NCCL ops cannot overlap the next step's activation p2p.
|
||||
self.disabled = False
|
||||
|
||||
def set_disabled(self, disabled: bool) -> None:
|
||||
self.disabled = disabled
|
||||
|
||||
def on_req_idx_freed(self, req_idx: int) -> None:
|
||||
self.req_idx_gen_np[req_idx] += 1
|
||||
|
||||
@@ -157,29 +177,15 @@ class PPHandler:
|
||||
num_sampled=slot.num_sampled,
|
||||
num_rejected=slot.num_rejected,
|
||||
idx_mapping=idx_mapping,
|
||||
broadcast_drafts=slot.draft_tokens,
|
||||
)
|
||||
|
||||
def broadcast_drafts(
|
||||
self, draft_tokens: torch.Tensor, input_batch: InputBatch
|
||||
) -> None:
|
||||
"""Broadcast draft proposals so non-last ranks can embed real token ids."""
|
||||
assert self.is_last_rank
|
||||
if compute_need_sampled_mask(input_batch) is None:
|
||||
return
|
||||
with torch.cuda.stream(self.broadcast_stream):
|
||||
self.broadcast_stream.wait_stream(self.main_stream)
|
||||
send = draft_tokens[input_batch.idx_mapping].contiguous()
|
||||
# Must record the idx_mapping tensor since it was allocated
|
||||
# on the main stream.
|
||||
input_batch.idx_mapping.record_stream(self.broadcast_stream)
|
||||
torch.distributed.broadcast(
|
||||
send, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
|
||||
def receive(self, input_batch: InputBatch) -> bool:
|
||||
"""Returns True iff sampled tokens need to be gathered from *all*
|
||||
requests in the batch."""
|
||||
assert not self.is_last_rank
|
||||
if self.disabled:
|
||||
return False
|
||||
need_sampled_mask = compute_need_sampled_mask(input_batch)
|
||||
if need_sampled_mask is None:
|
||||
# Leave this step's reserved slot as None.
|
||||
@@ -195,7 +201,7 @@ class PPHandler:
|
||||
sampled_tokens = torch.empty(
|
||||
num_reqs, self.max_sample_len, dtype=torch.int64, device=self.device
|
||||
)
|
||||
combined = torch.empty(2, num_reqs, dtype=torch.int32, device=self.device)
|
||||
combined = _alloc_combined(num_reqs, self.device)
|
||||
torch.distributed.broadcast(
|
||||
sampled_tokens, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
@@ -213,6 +219,7 @@ class PPHandler:
|
||||
torch.distributed.broadcast(
|
||||
draft_tokens, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
|
||||
event = self.broadcast_stream.record_event()
|
||||
num_sampled, num_rejected = combined.unbind(dim=0)
|
||||
# Must record_stream since these were allocated on broadcast stream but
|
||||
@@ -242,11 +249,16 @@ class PPHandler:
|
||||
input_batch: InputBatch,
|
||||
) -> None:
|
||||
assert self.is_last_rank
|
||||
if compute_need_sampled_mask(input_batch) is None:
|
||||
if self.disabled:
|
||||
return
|
||||
mask = compute_need_sampled_mask(input_batch)
|
||||
if mask is None:
|
||||
# No request needs sampled outputs for a subsequent decode step.
|
||||
return
|
||||
|
||||
assert sampled_token_ids.dtype == torch.int64
|
||||
assert num_sampled.dtype == torch.int32
|
||||
assert num_rejected.dtype == torch.int32
|
||||
|
||||
if current_platform.is_xpu():
|
||||
self.main_stream.synchronize()
|
||||
@@ -262,9 +274,42 @@ class PPHandler:
|
||||
src=self.last_rank,
|
||||
group=self.broadcast_group,
|
||||
)
|
||||
combined = torch.stack((num_sampled, num_rejected), dim=0)
|
||||
combined = _alloc_combined(num_sampled.shape[0], self.device)
|
||||
combined[0, : num_sampled.shape[0]] = num_sampled
|
||||
combined[1, : num_sampled.shape[0]] = num_rejected
|
||||
torch.distributed.broadcast(
|
||||
combined, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
for tensor in (sampled_token_ids, num_sampled, num_rejected):
|
||||
tensor.record_stream(self.broadcast_stream)
|
||||
|
||||
def broadcast_drafts(
|
||||
self, draft_token_table: torch.Tensor, input_batch: InputBatch
|
||||
) -> None:
|
||||
"""Broadcast the speculator's freshly proposed draft tokens.
|
||||
|
||||
Runs after propose() on the last rank; the send is stream-ordered
|
||||
after broadcast()'s sends, matching receive()'s enqueue order. The
|
||||
payload is gathered from the runner's draft table (just updated from
|
||||
propose()'s output) into a fresh compact tensor: the speculator
|
||||
overwrites its own persistent buffer on the next step, possibly
|
||||
before this async send completes.
|
||||
"""
|
||||
assert self.is_last_rank
|
||||
if self.disabled or self.max_sample_len == 1:
|
||||
return
|
||||
if compute_need_sampled_mask(input_batch) is None:
|
||||
return
|
||||
assert draft_token_table.dtype == torch.int64
|
||||
assert draft_token_table.shape[1] == self.max_sample_len - 1
|
||||
|
||||
# Gather on the main stream so the payload is ordered after this
|
||||
# step's table update and before any later one.
|
||||
drafts = draft_token_table[input_batch.idx_mapping]
|
||||
assert drafts.shape[0] == input_batch.num_reqs
|
||||
with torch.cuda.stream(self.broadcast_stream):
|
||||
self.broadcast_stream.wait_stream(self.main_stream)
|
||||
torch.distributed.broadcast(
|
||||
drafts, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
drafts.record_stream(self.broadcast_stream)
|
||||
|
||||
@@ -256,6 +256,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
slot_mapping=slot_mappings,
|
||||
batch_descriptor=batch_descriptor,
|
||||
is_padding=self.input_buffers.is_padding[:num_tokens],
|
||||
):
|
||||
last_hidden_states = self.model(
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens],
|
||||
@@ -487,6 +488,7 @@ def _prepare_dflash_inputs_kernel(
|
||||
# Outputs
|
||||
out_input_ids_ptr,
|
||||
out_query_positions_ptr,
|
||||
out_is_padding_ptr,
|
||||
out_query_start_loc_ptr,
|
||||
out_seq_lens_ptr,
|
||||
out_query_slot_mapping_ptr,
|
||||
@@ -617,6 +619,7 @@ def _prepare_dflash_inputs_kernel(
|
||||
)
|
||||
|
||||
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
|
||||
tl.store(out_is_padding_ptr + query_idx, False, mask=is_query)
|
||||
clamped_query_pos = tl.minimum(query_pos, max_model_len - 1)
|
||||
tl.store(out_query_positions_ptr + query_idx, clamped_query_pos, mask=is_query)
|
||||
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)
|
||||
@@ -679,6 +682,9 @@ def _prepare_dflash_inputs_kernel(
|
||||
for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE):
|
||||
block = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < max_num_tokens
|
||||
tl.store(out_input_ids_ptr + block, 0, mask=mask)
|
||||
tl.store(out_query_positions_ptr + block, 0, mask=mask)
|
||||
tl.store(out_is_padding_ptr + block, True, mask=mask)
|
||||
tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask)
|
||||
|
||||
|
||||
@@ -730,6 +736,7 @@ def prepare_dflash_inputs(
|
||||
_prepare_dflash_inputs_kernel[(num_reqs, num_blocks)](
|
||||
input_buffers.input_ids,
|
||||
input_buffers.positions,
|
||||
input_buffers.is_padding,
|
||||
input_buffers.query_start_loc,
|
||||
input_buffers.seq_lens,
|
||||
query_slot_mapping,
|
||||
|
||||
@@ -445,6 +445,11 @@ def _warmup_kernels(
|
||||
for step_indices, step_spec_flags in decode_steps:
|
||||
_run_decode_step(step_indices, step_spec_flags)
|
||||
|
||||
# The deferred PP post-update path only runs on real steps, so the steps
|
||||
# above never JIT-compile its kernel on non-last ranks.
|
||||
if not model_runner.is_last_pp_rank and model_runner.pp_handler is not None:
|
||||
model_runner.warmup_pp_decode_update()
|
||||
|
||||
# Clean up - process finish_req_ids.
|
||||
cleanup_output = SchedulerOutput.make_empty()
|
||||
cleanup_output.finished_req_ids = set(req_ids)
|
||||
|
||||
@@ -805,6 +805,15 @@ class Worker(WorkerBase):
|
||||
|
||||
@instrument(span_name="Warmup (GPU)")
|
||||
def compile_or_warm_up_model(self) -> CompilationTimes:
|
||||
# All warmup phases below run synthetic steps whose sampled outputs are
|
||||
# discarded. The PP sampled-token broadcast would carry no payload, and
|
||||
# its side-stream NCCL ops can overlap the next step's activation p2p
|
||||
# and deadlock the pipeline, so keep it disabled for the whole warmup
|
||||
# window and restore it before serving.
|
||||
pp_handler = getattr(self.model_runner, "pp_handler", None)
|
||||
if pp_handler is not None:
|
||||
pp_handler.set_disabled(True)
|
||||
|
||||
warmup_sizes: list[int] = []
|
||||
|
||||
if (
|
||||
@@ -983,6 +992,9 @@ class Worker(WorkerBase):
|
||||
# gate so subsequent `execute_model` / `sample_tokens` calls enforce it.
|
||||
enable_gpu_sync_check()
|
||||
|
||||
if pp_handler is not None:
|
||||
pp_handler.set_disabled(False)
|
||||
|
||||
return CompilationTimes(
|
||||
language_model=self.compilation_config.compilation_time,
|
||||
encoder=self.compilation_config.encoder_compilation_time,
|
||||
|
||||
Reference in New Issue
Block a user