diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index bb991888e2..8bd434a355 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1818,6 +1818,48 @@ steps: commands: - pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py +- label: ":amd: (MI355) Kimi K3" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/models/kimi_k3/ + - csrc/libtorch_stable/kimi_k3/ + - tests/models/kimi_k3/ + - vllm/platforms/rocm.py + commands: + # Mirrors the ":amd: (MI355) Kimi K3" step generated from + # .buildkite/test_areas/models_basic.yaml. The ROCm KDA/AttnRes kernels are + # gated on gfx950, so this has to be MI355. The two kernels/ files the NVIDIA + # step runs are CUDA-only (they import cuda.bindings.driver), so they are + # deliberately left out here. + - pytest -v -s models/kimi_k3 + +- label: ":amd: (MI355) Kimi K3 Multi-GPU" # TBD + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/models/kimi_k3/ + - tests/models/kimi_k3/ + - vllm/platforms/rocm.py + commands: + # The single-GPU step above skips every @multi_gpu_test case. `-m "distributed"` + # selects exactly those: multi_gpu_test() applies pytest.mark.distributed + # alongside its skipif (tests/utils.py:2080), so this picks up a new + # multi-GPU test anywhere in the directory without editing this list. + # Four GPUs run the two tp4 cases in test_amd_latent_moe_runner.py; the tp8 + # case and the 8/16-GPU cases in test_latent_moe_tail.py still skip. + - pytest -v -s -m "distributed" models/kimi_k3 + - label: ":amd: (MI300) Mamba Kernels" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -3562,6 +3604,49 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +- label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - tests/v1/kv_connector/rocm_pd_accuracy_utils.py + - tests/utils.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/platforms/rocm.py + - requirements/common.txt + - requirements/kv_connectors_rocm.txt + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py + +- label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/mooncake/ + - tests/v1/kv_connector/mooncake_integration/ + - tests/v1/kv_connector/rocm_pd_accuracy_utils.py + - tests/utils.py + - examples/disaggregated/mooncake_connector/ + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/platforms/rocm.py + - requirements/common.txt + - requirements/kv_connectors_rocm.txt + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py + - label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" # TBD timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index e849a65ab4..1258336009 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -42,6 +42,29 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + mirror: + amd: + label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 60 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - tests/v1/kv_connector/rocm_pd_accuracy_utils.py + - tests/utils.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/platforms/rocm.py + - requirements/common.txt + - requirements/kv_connectors_rocm.txt + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # Preserve all six model/TP pairs with native AITER attention. + - pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py - label: ":nvidia: (L4) Push NixlConnector PP prefill PD accuracy" key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus diff --git a/.buildkite/test_areas/disaggregated_mooncake.yaml b/.buildkite/test_areas/disaggregated_mooncake.yaml index d89996c9a2..030351ba57 100644 --- a/.buildkite/test_areas/disaggregated_mooncake.yaml +++ b/.buildkite/test_areas/disaggregated_mooncake.yaml @@ -19,6 +19,30 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh + mirror: + amd: + label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/mooncake/ + - tests/v1/kv_connector/mooncake_integration/ + - tests/v1/kv_connector/rocm_pd_accuracy_utils.py + - tests/utils.py + - examples/disaggregated/mooncake_connector/ + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/platforms/rocm.py + - requirements/common.txt + - requirements/kv_connectors_rocm.txt + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # Same full accuracy/TP sweep, using same-host HIP GPU transport. + - pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py - label: ":nvidia: (L4) Mooncake EC TCP E2E" key: mooncake-ec-tcp-e2e-2-gpus diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 9e0a8399a3..075b3733ca 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -12,6 +12,7 @@ steps: working_dir: "." env: SNAPSHOT_E2E_RUN_LIMIT_S: "1200" + NVIDIA_VISIBLE_DEVICES: "0" source_file_dependencies: - .buildkite/scripts/initialized-snapshot-e2e.sh - .buildkite/test_areas/entrypoints.yaml diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index f1ef63b4ed..9d27b2e83a 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -571,7 +571,7 @@ steps: - label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY" key: lm-eval-humming-h100-act optional: true - timeout_in_minutes: 70 + timeout_in_minutes: 90 device: h100 num_devices: 1 source_file_dependencies: @@ -582,7 +582,7 @@ steps: - vllm/model_executor/kernels/linear/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt - # - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt - label: ":nvidia: (H100) LM Eval Humming DeepEP TEMPORARY" key: lm-eval-humming-h100-deepep diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 930c2dd590..6f570b3459 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -109,6 +109,23 @@ steps: commands: # The native NVIDIA Kimi K3 kernels require the SM100 family. - pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py + mirror: + amd: + label: ":amd: (MI355) Kimi K3" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 60 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py + commands: + # The ROCm KDA/AttnRes kernels are gated on gfx950, so this has to be MI355. + # The two kernels/ files the NVIDIA job adds are CUDA-only (they import + # cuda.bindings.driver), so they are deliberately left out here. + - pytest -v -s models/kimi_k3 - label: ":nvidia: (H200) GLM5Next" key: glm5next-unit-tests diff --git a/csrc/cpu/cpu_attn.cpp b/csrc/cpu/cpu_attn.cpp index ea0b4e49b2..13e430fb41 100644 --- a/csrc/cpu/cpu_attn.cpp +++ b/csrc/cpu/cpu_attn.cpp @@ -243,7 +243,19 @@ void cpu_attention_with_kv_cache( input.block_table = block_table.data_ptr(); input.alibi_slopes = alibi_slopes.has_value() ? alibi_slopes->data_ptr() : nullptr; - input.s_aux = s_aux.has_value() ? s_aux->data_ptr() : nullptr; + // Attention sinks may be bf16 (native path) or fp32. Anything that is not + // bf16 must be provided as fp32 and is executed in full float precision. + if (s_aux.has_value()) { + TORCH_CHECK(s_aux->scalar_type() == at::ScalarType::BFloat16 || + s_aux->scalar_type() == at::ScalarType::Float, + "cpu_attention_with_kv_cache: s_aux (attention sinks) dtype ", + s_aux->scalar_type(), " must be bfloat16 or float32"); + input.s_aux = s_aux->data_ptr(); + input.s_aux_is_bf16 = s_aux->scalar_type() == at::ScalarType::BFloat16; + } else { + input.s_aux = nullptr; + input.s_aux_is_bf16 = false; + } input.dynamic_causal = dynamic_causal.has_value() ? dynamic_causal->data_ptr() : nullptr; input.scale = scale; diff --git a/csrc/cpu/cpu_attn_impl.hpp b/csrc/cpu/cpu_attn_impl.hpp index 4ade47eff3..d914e15063 100644 --- a/csrc/cpu/cpu_attn_impl.hpp +++ b/csrc/cpu/cpu_attn_impl.hpp @@ -879,7 +879,11 @@ struct AttentionInput { int32_t* seq_lens; int32_t* block_table; float* alibi_slopes; - c10::BFloat16* s_aux; + // Attention sinks pointer. May reference bf16 or fp32 data depending on + // `s_aux_is_bf16`. bf16 sinks keep the native bf16 path; anything else is + // provided as fp32 and executed in full float precision. + const void* s_aux; + bool s_aux_is_bf16; bool* dynamic_causal; float scale; bool causal; @@ -1565,7 +1569,8 @@ class AttentionMainLoop { const float scale = input->scale; const float softcap_scale = input->softcap; const float* alibi_slopes = input->alibi_slopes; - const c10::BFloat16* s_aux = input->s_aux; + const void* s_aux = input->s_aux; + const bool s_aux_is_bf16 = input->s_aux_is_bf16; const bool* dynamic_causal = input->dynamic_causal; const bool is_dynamic_causal = dynamic_causal != nullptr; @@ -1773,9 +1778,6 @@ class AttentionMainLoop { const float* curr_alibi_slopes = (alibi_slopes != nullptr ? alibi_slopes + q_head_start_idx : nullptr); - const c10::BFloat16* curr_s_aux = - (s_aux != nullptr ? s_aux + q_head_start_idx : nullptr); - // copy the Q tile to q_buffer, the logical layout of q_buffer is // [actual_q_token_num, curr_q_heads_per_kv, head_dim] { @@ -1787,10 +1789,23 @@ class AttentionMainLoop { if (use_sink) { alignas(64) float s_aux_fp32[16]; - // All other platforms have BF16Vec16 available - vec_op::BF16Vec16 vec_bf16(curr_s_aux); - vec_op::FP32Vec16 vec_fp32(vec_bf16); - vec_fp32.save(s_aux_fp32); + // Sinks may be provided as bf16 or fp32. Either way they are + // loaded into fp32: bf16 sinks are up-converted (native + // BF16Vec16 path, available on all platforms), while fp32 + // sinks are loaded directly and executed in full precision. + if (s_aux_is_bf16) { + const c10::BFloat16* curr_s_aux = + static_cast(s_aux) + + q_head_start_idx; + vec_op::BF16Vec16 vec_bf16(curr_s_aux); + vec_op::FP32Vec16 vec_fp32(vec_bf16); + vec_fp32.save(s_aux_fp32); + } else { + const float* curr_s_aux = + static_cast(s_aux) + q_head_start_idx; + vec_op::FP32Vec16 vec_fp32(curr_s_aux); + vec_fp32.save(s_aux_fp32); + } float* __restrict__ curr_sum_buffer = sum_buffer; float* __restrict__ curr_max_buffer = max_buffer; diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index 789878f48a..7353bd3594 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -145,6 +145,9 @@ When batch invariance is enabled, vLLM: 1. Uses deterministic kernel implementations for attention and other operations 2. Ensures consistent numerical behavior across different batch sizes 3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode) +4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), + runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out + (not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes). !!! note Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility. diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index c6e1786ee0..adf5d2ea03 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -549,6 +549,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I+ | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ | | `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I+ | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ | | `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I+ | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | ✅︎ | ✅︎ | +| `DiffusionGemmaForBlockDiffusion` | DiffusionGemma | T + I+ + V+ | `google/diffusiongemma-26B-A4B-it`, etc. | | | | `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + IE+ | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ | | `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I+/ V+ | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ | | `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + IE+ | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ | diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 72609d8983..59e5c4dcc8 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -33,4 +33,4 @@ quack-kernels==0.6.5 # Required by MSA tokenspeed-mla==0.1.8; platform_system == "Linux" # Humming kernels for quantization gemm -humming-kernels[cu13]==0.1.15 +humming-kernels[cu13]==0.1.16 diff --git a/tests/compile/fusions_e2e/common.py b/tests/compile/fusions_e2e/common.py index 77c3ba5112..1f92ff5a19 100644 --- a/tests/compile/fusions_e2e/common.py +++ b/tests/compile/fusions_e2e/common.py @@ -128,8 +128,11 @@ FUSION_LOG_PATTERNS: dict[str, re.Pattern] = { ), "rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"), "act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"), + # The combined pass can consume these sites before the standalone pass. "norm_rope_fusion": re.compile( - r"qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on (\d+) sites" + r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|" + r"qk_norm_rope_kvcache_fusion.py:\d+] " + r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)" ), "attn_quant_fusion": re.compile( r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes" diff --git a/tests/compile/fusions_e2e/conftest.py b/tests/compile/fusions_e2e/conftest.py index f3df3c358e..a2cbe0135b 100644 --- a/tests/compile/fusions_e2e/conftest.py +++ b/tests/compile/fusions_e2e/conftest.py @@ -182,7 +182,11 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn): # TODO: Remove log counting in unit tests # once all matchers implement VllmFusionPatternMatcherPass n_expected = tp_size * num_ranges_activated - if match_name not in ("attn_quant_fusion", "act_quant_fusion"): + if match_name not in ( + "attn_quant_fusion", + "act_quant_fusion", + "norm_rope_fusion", + ): assert len(log_matches) == n_expected, ( f"Could not find {n_expected} {match_name} " f"(found {len(log_matches)}) in:\n {log_holder.text}" @@ -190,7 +194,16 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn): expected_matches = getattr(matches, match_name) - if match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check: + if match_name == "norm_rope_fusion": + # Opaque LayerName lets the combined pass consume short-range + # sites before the standalone pass; count either owner. + assert len(log_matches) >= n_expected + assert all(m in (0, expected_matches) for m in log_matches) + assert sum(log_matches) == expected_matches * n_expected, ( + f"{match_name} expected {expected_matches * n_expected} " + f"sites across both passes, found: {log_matches}" + ) + elif match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check: # AR+rms+quant takes precedence over rms+quant if activated. # That means we get full matching where ar+rms+quant was not # activated, and less where it was (only the smallest range). diff --git a/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py b/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py index 6965346a58..ee557235fd 100644 --- a/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py +++ b/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py @@ -8,6 +8,7 @@ import pytest import torch from packaging.version import Version +import vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion as fusion_module import vllm.config from tests.compile.backend import TestBackend from tests.v1.attention.utils import ( @@ -15,8 +16,12 @@ from tests.v1.attention.utils import ( create_common_attn_metadata, dense_kv_cache_views, ) -from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops -from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP +from vllm._aiter_ops import ( + IS_AITER_FOUND, + is_aiter_found_and_supported, + rocm_aiter_ops, +) +from vllm.compilation.passes.fusion.matcher_utils import MROPE_OP, ROTARY_OP from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import ( QkNormRopeKvCacheFusionPass, ) @@ -37,8 +42,12 @@ from vllm.config import ( from vllm.forward_context import get_forward_context, set_forward_context from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig +from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding +from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding from vllm.platforms import current_platform +from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name from vllm.v1.attention.backend import ( AttentionBackend, CommonAttentionMetadata, @@ -94,6 +103,7 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module): device: torch.device, rotary_dim: int | None = None, prefix: str = "model.layers.0.self_attn.attn", + expected_query_quant_group_shape: GroupShape | None = None, ): super().__init__() self.num_heads = num_heads @@ -133,6 +143,9 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module): attn_backend=attn_backend.get_class(), ) self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend() + if expected_query_quant_group_shape is not None: + assert self.attn.query_quant is not None + assert self.attn.query_quant.group_shape == expected_query_quant_group_shape assert not self.attn_backend.forward_includes_kv_cache_update, ( f"Attention backend {self.attn_backend} does not support " "fuse_qk_norm_rope_kvcache." @@ -227,18 +240,15 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module): self.kv_cache_dtype != self.dtype and self.attn.impl.supports_quant_query_input ): - q_fp8 = torch.empty_like(q, dtype=FP8_DTYPE) - torch.ops.vllm.rocm_aiter_per_tensor_quant( - q_fp8, q, self.attn._q_scale, False - ) - q = q_fp8 + assert self.attn.query_quant is not None + q, _ = self.attn.query_quant(q, self.attn._q_scale) # Final views + KV cache update q = q.view(-1, self.num_heads, self.head_size) k = k.view(-1, self.num_kv_heads, self.head_size) v = v.view(-1, self.num_kv_heads, self.head_size) kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( - k, v, self.layer_name + k, v, _encode_layer_name(self.layer_name) ) return q, k, v, kv_cache_dummy_dep @@ -260,6 +270,64 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module): return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default] +class QKNormMRoPEKVCacheTestModel(QKNormRoPEKVCacheTestModel): + def __init__( + self, + *, + mrope_section: tuple[int, int, int], + mrope_interleaved: bool, + **kwargs, + ): + super().__init__(**kwargs) + self.rotary_emb = MRotaryEmbedding( + self.head_size, + rotary_dim=self.rotary_dim, + max_position_embeddings=4096, + base=10000, + is_neox_style=self.is_neox, + dtype=self.dtype, + mrope_section=list(mrope_section), + mrope_interleaved=mrope_interleaved, + ) + self.enable_rope_custom_op = self.rotary_emb.enabled() + + def forward( + self, qkv: torch.Tensor, positions: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + q, k, v, kv_cache_dummy_dep = super().forward(qkv, positions) + output = torch.empty( + q.shape[0], + self.num_heads * self.head_size, + device=q.device, + dtype=self.dtype, + ) + torch.ops.vllm.unified_attention_with_output( + q, + k, + v, + output, + _encode_layer_name(self.layer_name), + kv_cache_dummy_dep=kv_cache_dummy_dep, + ) + # Production decoder attention consumes K from the cache rather than + # its direct K argument. Returning `output` keeps that production user + # in the graph without exposing K to an unsupported consumer. + return q, output, v, kv_cache_dummy_dep + + def ops_in_model_before(self) -> list[torch._ops.OpOverload]: + return [MROPE_OP, torch.ops.vllm.unified_kv_cache_update.default] + + def ops_in_model_after(self) -> list[torch._ops.OpOverload]: + return [torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default] + + +class QKNormMRoPEObservableKTestModel(QKNormMRoPEKVCacheTestModel): + def forward( + self, qkv: torch.Tensor, positions: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + return QKNormRoPEKVCacheTestModel.forward(self, qkv, positions) + + def _run_qk_norm_rope_kvcache_fusion_test( *, attn_backend: AttentionBackendEnum, @@ -278,18 +346,33 @@ def _run_qk_norm_rope_kvcache_fusion_test( rms_norm_eps: float, custom_op: str, monkeypatch: pytest.MonkeyPatch, + mrope_section: tuple[int, int, int] | None = None, + mrope_interleaved: bool = False, + observe_mrope_k: bool = False, + expect_fusion: bool = True, + use_quark_scalar_query_scale: bool = False, ) -> None: device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda") torch.set_default_device(device) torch.set_default_dtype(dtype) torch.manual_seed(0) + # Set AITER toggles before VllmConfig initialization enables env caching. + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1") + rocm_aiter_ops.refresh_env_variables() + vllm_config = VllmConfig( model_config=ModelConfig(dtype=dtype), cache_config=CacheConfig( block_size=block_size, cache_dtype=kv_cache_dtype, ), + quant_config=( + QuarkConfig(quant_config={"exclude": []}) + if use_quark_scalar_query_scale + else None + ), compilation_config=CompilationConfig( mode=CompilationMode.VLLM_COMPILE, custom_ops=[custom_op], @@ -302,6 +385,7 @@ def _run_qk_norm_rope_kvcache_fusion_test( with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m: m.setenv("VLLM_ROCM_USE_AITER", "1") + m.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1") m.setenv( "VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0", @@ -309,18 +393,49 @@ def _run_qk_norm_rope_kvcache_fusion_test( m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout) rocm_aiter_ops.refresh_env_variables() - model = QKNormRoPEKVCacheTestModel( - vllm_config=vllm_config, - attn_backend=attn_backend, - num_heads=num_heads, - num_kv_heads=num_kv_heads, - head_size=head_size, - rotary_dim=rotary_dim, - is_neox=is_neox, - rms_norm_eps=rms_norm_eps, - dtype=dtype, - device=torch.get_default_device(), - ) + expected_query_quant_group_shape = None + if use_quark_scalar_query_scale: + expected_query_quant_group_shape = ( + GroupShape(-1, num_heads * head_size) + if num_kv_heads == 1 + else GroupShape.PER_TENSOR + ) + + model_kwargs = { + "vllm_config": vllm_config, + "attn_backend": attn_backend, + "num_heads": num_heads, + "num_kv_heads": num_kv_heads, + "head_size": head_size, + "rotary_dim": rotary_dim, + "is_neox": is_neox, + "rms_norm_eps": rms_norm_eps, + "dtype": dtype, + "device": torch.get_default_device(), + "expected_query_quant_group_shape": expected_query_quant_group_shape, + } + if mrope_section is None: + model = QKNormRoPEKVCacheTestModel(**model_kwargs) + else: + alternate_section = (8, 8, 8) + m.setattr( + fusion_module, + "_discover_mrope_configs", + lambda _: ( + (mrope_section, mrope_interleaved), + (alternate_section, not mrope_interleaved), + ), + ) + model_cls = ( + QKNormMRoPEObservableKTestModel + if observe_mrope_k + else QKNormMRoPEKVCacheTestModel + ) + model = model_cls( + **model_kwargs, + mrope_section=mrope_section, + mrope_interleaved=mrope_interleaved, + ) fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config) passes = [ @@ -337,7 +452,12 @@ def _run_qk_norm_rope_kvcache_fusion_test( num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype, ) - pos = torch.arange(num_tokens, dtype=torch.long) + if mrope_section is None: + pos = torch.arange(num_tokens, dtype=torch.long) + else: + # Real Qwen-VL position IDs can be a strided [3, N] view. + pos = torch.randint(0, 4096, (3, 2 * num_tokens), dtype=torch.long)[:, ::2] + assert not pos.is_contiguous() qkv_unfused = qkv.clone() pos_unfused = pos.clone() @@ -356,7 +476,7 @@ def _run_qk_norm_rope_kvcache_fusion_test( # Run fused (compiled) forward torch._dynamo.mark_dynamic(qkv, 0) - torch._dynamo.mark_dynamic(pos, 0) + torch._dynamo.mark_dynamic(pos, 0 if mrope_section is None else 1) with set_forward_context(None, vllm_config): model_fused = torch.compile(model, backend=backend) forward_context = get_forward_context() @@ -369,14 +489,33 @@ def _run_qk_norm_rope_kvcache_fusion_test( kv_cache_fused = attn_layer.kv_cache del dummy - assert fusion_pass.matched_count == 1 + assert fusion_pass.matched_count == int(expect_fusion) - backend.check_before_ops(model.ops_in_model_before()) - backend.check_after_ops(model.ops_in_model_after()) + if expect_fusion: + backend.check_before_ops(model.ops_in_model_before()) + backend.check_after_ops(model.ops_in_model_after()) + else: + for op in model.ops_in_model_before(): + assert backend.op_count(op, before=True) > 0 + assert backend.op_count(op) > 0 + + if use_quark_scalar_query_scale: + expected_quant_op = ( + torch.ops.vllm.rocm_aiter_per_tensor_quant.default + if num_kv_heads != 1 + else torch.ops._C.static_scaled_fp8_quant.default + ) + assert backend.op_count(expected_quant_op, before=True) > 0 + # Query quantization remains separate after the prologue fusion. + assert backend.op_count(expected_quant_op) > 0 # Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2; # AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2. - ATOL, RTOL = (2e-2, 2e-2) if enable_aiter_triton_rope else (1e-2, 1e-2) + ATOL, RTOL = ( + (2e-2, 2e-2) + if enable_aiter_triton_rope or mrope_section is not None + else (1e-2, 1e-2) + ) is_fp8_cache = model.kv_cache_dtype != dtype if q_fused.dtype == FP8_DTYPE: @@ -387,7 +526,7 @@ def _run_qk_norm_rope_kvcache_fusion_test( else: torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL) - if not is_fp8_cache: + if not expect_fusion or (not is_fp8_cache and mrope_section is None): # The AITER PTS kernel populates k_out only for non-FP8 caches. # With FP8, the kernel writes quantized K directly to the cache # and may leave k_out uninitialised. In production this is fine @@ -397,15 +536,16 @@ def _run_qk_norm_rope_kvcache_fusion_test( # Should be bit exact since no processing had been done on v for both paths torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0) - # fp8 vs triton-rope ref requires loosening tolerance to 1.25e-1. - if is_fp8_cache and enable_aiter_triton_rope: + # Fused and unfused arithmetic can straddle an FP8 rounding boundary. + # Allow one E4M3 quantization step while still comparing every block. + if is_fp8_cache: cache_atol = cache_rtol = 1.25e-1 else: cache_atol, cache_rtol = ATOL, RTOL torch.testing.assert_close( - kv_cache_unfused[0].float(), - kv_cache_fused[0].float(), + kv_cache_unfused.float(), + kv_cache_fused.float(), atol=cache_atol, rtol=cache_rtol, ) @@ -494,3 +634,225 @@ def test_qk_norm_rope_kvcache_fusion( custom_op=custom_op, monkeypatch=monkeypatch, ) + + +@pytest.mark.parametrize( + "mrope_section", + [ + pytest.param((16, 24, 24), id="full_rotary"), + pytest.param((4, 6, 6), id="partial_rotary"), + ], +) +@pytest.mark.parametrize("mrope_interleaved", [False, True]) +@pytest.mark.parametrize("is_neox", [False, True]) +@pytest.mark.parametrize( + "kv_layout", + [ + pytest.param(KVCacheLayout.LBHNC, id="head_major"), + pytest.param(KVCacheLayout.LBNHC, id="token_major"), + pytest.param(KVCacheLayout.LHBNC, id="block_major"), + ], +) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +@pytest.mark.skipif( + not IS_AITER_FOUND, + reason="Requires AITER", +) +def test_qk_norm_mrope_kvcache_fusion( + mrope_section: tuple[int, int, int], + mrope_interleaved: bool, + is_neox: bool, + kv_layout: KVCacheLayout, + kv_cache_dtype: str, + monkeypatch: pytest.MonkeyPatch, +): + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + enable_aiter_triton_rope=False, + num_tokens=5, + num_heads=8, + num_kv_heads=2, + head_size=128, + rotary_dim=2 * sum(mrope_section), + block_size=16, + is_neox=is_neox, + use_shuffle_kv_layout="0", + kv_layout=kv_layout, + dtype=torch.bfloat16, + kv_cache_dtype=kv_cache_dtype, + rms_norm_eps=1e-6, + custom_op="+rotary_embedding", + monkeypatch=monkeypatch, + mrope_section=mrope_section, + mrope_interleaved=mrope_interleaved, + ) + + +@pytest.mark.parametrize( + ("num_heads", "num_kv_heads"), + [(32, 2), (16, 1), (8, 1)], + ids=["tp2", "tp4", "tp8"], +) +def test_qk_norm_mrope_kvcache_fusion_quark_scalar_query_scale( + num_heads: int, + num_kv_heads: int, + monkeypatch: pytest.MonkeyPatch, +): + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + enable_aiter_triton_rope=False, + num_tokens=5, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_size=128, + rotary_dim=128, + block_size=16, + is_neox=True, + use_shuffle_kv_layout="0", + kv_layout=KVCacheLayout.LBHNC, + dtype=torch.bfloat16, + kv_cache_dtype="fp8", + rms_norm_eps=1e-6, + custom_op="+rotary_embedding", + monkeypatch=monkeypatch, + mrope_section=(24, 20, 20), + mrope_interleaved=True, + use_quark_scalar_query_scale=True, + ) + + +def test_mrope_fusion_rejects_observable_k(monkeypatch: pytest.MonkeyPatch): + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + enable_aiter_triton_rope=False, + num_tokens=5, + num_heads=8, + num_kv_heads=2, + head_size=128, + rotary_dim=128, + block_size=16, + is_neox=True, + use_shuffle_kv_layout="0", + kv_layout=KVCacheLayout.LBHNC, + dtype=torch.bfloat16, + kv_cache_dtype="auto", + rms_norm_eps=1e-6, + custom_op="+rotary_embedding", + monkeypatch=monkeypatch, + mrope_section=(16, 24, 24), + observe_mrope_k=True, + expect_fusion=False, + ) + + +def test_mrope_fusion_rejects_unaligned_rotary_dim( + monkeypatch: pytest.MonkeyPatch, +): + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + enable_aiter_triton_rope=False, + num_tokens=5, + num_heads=8, + num_kv_heads=2, + head_size=128, + rotary_dim=6, + block_size=16, + is_neox=True, + use_shuffle_kv_layout="0", + kv_layout=KVCacheLayout.LBHNC, + dtype=torch.bfloat16, + kv_cache_dtype="auto", + rms_norm_eps=1e-6, + custom_op="+rotary_embedding", + monkeypatch=monkeypatch, + mrope_section=(1, 1, 1), + expect_fusion=False, + ) + + +@pytest.mark.skipif(not _USE_LAYERNAME, reason="requires opaque LayerName") +def test_opaque_layer_name_patterns_register_once(monkeypatch: pytest.MonkeyPatch): + device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda") + vllm_config = VllmConfig( + model_config=ModelConfig(dtype=torch.bfloat16), + cache_config=CacheConfig(block_size=16, cache_dtype="fp8_e4m3"), + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + custom_ops=["+rotary_embedding"], + pass_config=PassConfig(fuse_qk_norm_rope_kvcache=True), + ), + ) + + with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m: + m.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + model_kwargs = { + "vllm_config": vllm_config, + "attn_backend": AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + "num_heads": 8, + "num_kv_heads": 2, + "head_size": 128, + "is_neox": True, + "rms_norm_eps": 1e-6, + "dtype": torch.bfloat16, + "device": torch.device(device), + } + models = [ + QKNormRoPEKVCacheTestModel( + **model_kwargs, + prefix=f"model.layers.{index}.self_attn.attn", + ) + for index in range(2) + ] + + # Opaque layer names are wildcards in the pattern graph. Registering + # once per physical layer raises "Duplicate pattern" on torch >= 2.11. + fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config) + + assert models + assert fusion_pass.patterns + + +def _call_fused_mrope_impl(q_out: torch.Tensor) -> None: + device = q_out.device + fusion_module.fused_qk_norm_mrope_and_unified_kv_cache_update_impl( + q_out=q_out, + qkv=torch.zeros(2, 16, device=device), + positions=torch.arange(2, device=device), + q_weight=torch.ones(4, device=device), + k_weight=torch.ones(4, device=device), + rms_norm_eps=1e-6, + cos_sin_cache=torch.zeros(16, 4, device=device), + is_neox=True, + mrope_section_t=1, + mrope_section_h=1, + mrope_section_w=0, + is_interleaved=False, + rotary_dim=4, + layer_name="model.layers.0.self_attn.attn", + ) + + +def test_mrope_profiling_pass_initializes_query(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + fusion_module, + "get_attention_context", + lambda _: (None, None, None, None), + ) + q_out = torch.full((2, 8), torch.nan, device="cpu") + + _call_fused_mrope_impl(q_out) + + torch.testing.assert_close(q_out, torch.zeros_like(q_out)) + + +def test_mrope_attention_context_failure_propagates( + monkeypatch: pytest.MonkeyPatch, +): + def fail_context(_): + raise RuntimeError("missing attention context") + + monkeypatch.setattr(fusion_module, "get_attention_context", fail_context) + + with pytest.raises(RuntimeError, match="missing attention context"): + _call_fused_mrope_impl(torch.empty(2, 8, device="cpu")) diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 28a499e129..060ae01a1d 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -14,8 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.scale_out.render.serving import ServingRender +from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.exceptions import GenerationError, VLLMValidationError from vllm.outputs import CompletionOutput, RequestOutput +from vllm.renderers.deepseek_v4 import DeepseekV4Renderer from vllm.renderers.hf import HfRenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config @@ -24,6 +26,9 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats MODEL_NAME = "openai-community/gpt2" MODEL_NAME_SHORT = "gpt2" +DEEPSEEK_V4_FIM_BEGIN = "<|fim▁begin|>" +DEEPSEEK_V4_FIM_HOLE = "<|fim▁hole|>" +DEEPSEEK_V4_FIM_END = "<|fim▁end|>" _PER_REQUEST_STATS = RequestStateStats( queued_ts=1.0, scheduled_ts=1.5, @@ -302,6 +307,179 @@ def test_completion_metrics_carries_both_timing_and_spec_decode(): assert response.metrics.speculative_decoding.num_spec_steps == 2 +def _build_online_renderer_for_completion( + model_config: MockModelConfig, + renderer: Any | None = None, +) -> OnlineRenderer: + if renderer is None: + renderer = MagicMock() + renderer.render_completion_suffix.return_value = None + + return OnlineRenderer( + model_config=model_config, + renderer=renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + + +def _deepseek_v4_suffix_renderer( + model_config: MockModelConfig, +) -> DeepseekV4Renderer: + return DeepseekV4Renderer( + MockVllmConfig(model_config, parallel_config=MockParallelConfig()), + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_uses_renderer_capability(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}]) + + request = CompletionRequest( + model=MODEL_NAME, + prompt="def fib(n):\n return ", + suffix="\n\nprint(fib(10))", + max_tokens=64, + ) + + result = await online_renderer.render_completion(request) + + assert result == [{"ok": True}] + online_renderer.preprocess_completion.assert_awaited_once() + assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == ( + f"{DEEPSEEK_V4_FIM_BEGIN}def fib(n):\n return " + f"{DEEPSEEK_V4_FIM_HOLE}\n\nprint(fib(10))" + f"{DEEPSEEK_V4_FIM_END}" + ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_supports_text_prompt_list(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}]) + + request = CompletionRequest( + model=MODEL_NAME, + prompt=["prefix A", "prefix B"], + suffix=" suffix", + ) + + result = await online_renderer.render_completion(request) + + assert result == [{"ok": True}] + assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [ + f"{DEEPSEEK_V4_FIM_BEGIN}prefix A" + f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}", + f"{DEEPSEEK_V4_FIM_BEGIN}prefix B" + f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}", + ] + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_renderer_without_fim_support(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion(model_config) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "FIM completion rendering" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_echo(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + echo=True, + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "Echo is unsupported with suffix" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_prompt_embeds(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + prompt_embeds=b"embeds", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "suffix is not supported with prompt_embeds" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize("prompt", ([1, 2, 3], [[1, 2, 3]])) +async def test_completion_suffix_rejects_token_prompt(prompt): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt=prompt, + suffix="suffix", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "requires text prompt input" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + @pytest.mark.asyncio async def test_completion_error_non_stream(): """Test finish_reason='error' returns 500 InternalServerError (non-streaming).""" diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml index b83d9e6a9e..c0bb8d3240 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml @@ -3,6 +3,5 @@ accuracy_threshold: 0.39 num_questions: 1319 num_fewshot: 5 server_args: >- - --enforce-eager --max-model-len 8192 --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml index 8ca4777ce5..357e7a0257 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml @@ -3,7 +3,6 @@ accuracy_threshold: 0.86 num_questions: 1319 num_fewshot: 5 server_args: >- - --enforce-eager --max-model-len 8192 --moe-backend humming --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml index 502ec79522..e41aa130ae 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml @@ -1,5 +1,5 @@ -model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM" -accuracy_threshold: 0.78 +model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-a4" +accuracy_threshold: 0.70 num_questions: 1319 num_fewshot: 5 server_args: >- diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml new file mode 100644 index 0000000000..965a0fe54d --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml @@ -0,0 +1,8 @@ +model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-asym" +accuracy_threshold: 0.85 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 4096 + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt b/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt index 0537ed974a..0852d14633 100644 --- a/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt +++ b/tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt @@ -1,3 +1,4 @@ gpt-oss-20b-humming.yaml Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml +Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml diff --git a/tests/evals/gsm8k/configs/humming/config-act-int8.txt b/tests/evals/gsm8k/configs/humming/config-act-int8.txt index 68f5cd1c15..7a608c7c5a 100644 --- a/tests/evals/gsm8k/configs/humming/config-act-int8.txt +++ b/tests/evals/gsm8k/configs/humming/config-act-int8.txt @@ -4,4 +4,4 @@ Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml Qwen3-30B-A3B-AWQ-humming.yaml Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml -Qwen3-4B-mixed-quant-RTN-humming.yaml +Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml diff --git a/tests/kernels/attention/test_cpu_attn.py b/tests/kernels/attention/test_cpu_attn.py index 7c4b7811c7..3046e2d2ac 100644 --- a/tests/kernels/attention/test_cpu_attn.py +++ b/tests/kernels/attention/test_cpu_attn.py @@ -428,6 +428,7 @@ def varlen_with_paged_kv( k_scale: float = 1.0, v_scale: float = 1.0, dynamic_causal: list[bool] | None = None, + s_aux_dtype: torch.dtype = torch.bfloat16, ) -> None: set_random_seed(0) num_seqs = len(seq_lens) @@ -449,9 +450,7 @@ def varlen_with_paged_kv( # 2^(-8/n) alibi_slopes = _get_alibi_slopes(num_query_heads) if use_alibi else None - s_aux = ( - 15 * torch.rand((num_query_heads,), dtype=torch.bfloat16) if use_sink else None - ) + s_aux = 15 * torch.rand((num_query_heads,), dtype=s_aux_dtype) if use_sink else None is_fp8 = kv_cache_dtype != "auto" if is_fp8 and current_platform.get_cpu_architecture() != CpuArchEnum.X86: @@ -538,6 +537,8 @@ def varlen_with_paged_kv( ) out_without_split = torch.empty_like(query) + if s_aux is not None and s_aux.dtype != torch.bfloat16: + s_aux = s_aux.to(torch.float32) cpu_attention_with_kv_cache( query=query, key_cache=packed_key_cache, @@ -1128,6 +1129,51 @@ def test_varlen_with_paged_kv_sink( ) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_e4m3"]) +@pytest.mark.parametrize("seq_lens", SEQ_LENS) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize("head_size", [96]) +@pytest.mark.parametrize("block_size", [128]) +@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("soft_cap", [None]) +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS) +@pytest.mark.parametrize("use_alibi", [False]) +@pytest.mark.parametrize("use_sink", [True]) +@pytest.mark.parametrize("isa", [get_attn_isa()]) +@pytest.mark.parametrize("s_aux_dtype", [torch.float16]) +def test_varlen_with_paged_kv_sink_fp16( + seq_lens: list[tuple[int, int]], + num_heads: tuple[int, int], + head_size: int, + sliding_window: int | None, + dtype: torch.dtype, + block_size: int, + soft_cap: float | None, + num_blocks: int, + use_alibi: bool, + use_sink: bool, + isa: str, + kv_cache_dtype: str, + s_aux_dtype: torch.dtype, +) -> None: + varlen_with_paged_kv( + seq_lens=seq_lens, + num_heads=num_heads, + head_size=head_size, + sliding_window=sliding_window, + dtype=dtype, + block_size=block_size, + soft_cap=soft_cap, + num_blocks=num_blocks, + use_alibi=use_alibi, + use_sink=use_sink, + isa=isa, + kv_cache_dtype=kv_cache_dtype, + s_aux_dtype=s_aux_dtype, + ) + + @pytest.mark.parametrize( "kv_cache_dtype", [ diff --git a/tests/kernels/moe/test_block_fp8.py b/tests/kernels/moe/test_block_fp8.py index 63f64c735a..68f6b3b0b3 100644 --- a/tests/kernels/moe/test_block_fp8.py +++ b/tests/kernels/moe/test_block_fp8.py @@ -107,7 +107,16 @@ SEEDS = [0] def torch_w8a8_block_fp8_moe( - a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False + a, + w1, + w2, + w1_s, + w2_s, + topk_weight, + topk_ids, + block_shape, + silu_fp32=False, + expert_map=None, ): """Fused MoE with block-wise fp8 quantization using native torch. @@ -123,6 +132,9 @@ def torch_w8a8_block_fp8_moe( topk_weight = topk_weight.view(-1) topk_ids = topk_ids.view(-1) + if expert_map is not None: + # w1/w2 hold this rank's experts; slots mapped to -1 contribute nothing. + topk_ids = expert_map[topk_ids] _, block_k = block_shape[0], block_shape[1] # Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the @@ -301,6 +313,54 @@ def test_w8a8_block_fp8_clamp_skips_fused_silu_quant( ) +@pytest.mark.parametrize(("M", "N", "K"), [(83, 512, 512), (2048, 4608, 512)]) +@pytest.mark.parametrize("topk", [2, 6]) +@pytest.mark.parametrize("ep_size", [2, 4]) +@torch.inference_mode() +def test_w8a8_block_fp8_fused_moe_expert_map(M, N, K, topk, ep_size, workspace_init): + """One EP rank: local expert shard plus an expert_map with -1 entries.""" + E, block_size, dtype = 16, [128, 128], torch.bfloat16 + torch.manual_seed(0) + + a = torch.randn((M, K), dtype=dtype) / 10 + score = torch.randn((M, E), dtype=dtype) + w1, w2, quant_config = make_test_quant_config( + E, N, K, dtype, quant_dtype=torch.float8_e4m3fn, block_shape=block_size + ) + + local_e = E // ep_size + e_ids = torch.randperm(E, dtype=torch.int32)[:local_e] + e_map = torch.full((E,), -1, dtype=torch.int32) + e_map[e_ids] = torch.arange(local_e, dtype=torch.int32) + w1, w2 = w1[e_ids].contiguous(), w2[e_ids].contiguous() + w1_s = quant_config.w1_scale[e_ids].contiguous() + w2_s = quant_config.w2_scale[e_ids].contiguous() + quant_config = fp8_w8a8_moe_quant_config( + w1_scale=w1_s, w2_scale=w2_s, block_shape=block_size + ) + + m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config) + topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False) + + with set_current_vllm_config(vllm_config): + ref_out = torch_w8a8_block_fp8_moe( + a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size, expert_map=e_map + ) + m_out = m_fused_moe.apply( + a, + w1, + w2, + topk_weights, + topk_ids, + activation=MoEActivation.SILU, + apply_router_weight_on_input=False, + expert_map=e_map, + global_num_experts=E, + ) + + torch.testing.assert_close(m_out, ref_out, atol=0.035, rtol=0.035) + + @pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG) @pytest.mark.parametrize("E", E) @pytest.mark.parametrize("topk", TOP_KS) diff --git a/tests/kernels/moe/test_topk_softplus_sqrt.py b/tests/kernels/moe/test_topk_softplus_sqrt.py index bc438c6693..bb8856f83a 100644 --- a/tests/kernels/moe/test_topk_softplus_sqrt.py +++ b/tests/kernels/moe/test_topk_softplus_sqrt.py @@ -339,6 +339,136 @@ def test_dsv4_fast_topk( ) +@pytest.mark.skipif( + not current_platform.is_cuda(), + reason="The DeepSeek V4 fast path is CUDA-only.", +) +def test_dsv4_fast_topk_padding_uint32_falls_back(monkeypatch: pytest.MonkeyPatch): + """Padded rows need the -1 sentinel, which uint32 cannot represent: the + router must skip the dsv4 fast path and still route the real rows.""" + torch.manual_seed(0) + num_tokens = 17 + num_experts = 256 + hidden_states = torch.randn((num_tokens, 64), dtype=torch.float32, device="cuda") + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.float32, device="cuda" + ) + correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda") + is_padding = torch.zeros(num_tokens, dtype=torch.bool, device="cuda") + is_padding[1::2] = True + gating_output[is_padding] = float("nan") + + monkeypatch.setattr( + "vllm.model_executor.layers.fused_moe.router." + "fused_topk_bias_router._get_padding_mask", + lambda _: is_padding, + ) + # uint32 + padding would trip dsv4_topk's signed-indices assertion; the + # generic path must take over instead. + topk_weights, topk_ids = fused_topk_bias( + hidden_states=hidden_states, + gating_output=gating_output, + scoring_func="sqrtsoftplus", + e_score_correction_bias=correction_bias, + topk=6, + renormalize=True, + indices_type=torch.uint32, + routed_scaling_factor=1.5, + ) + + assert topk_ids.dtype == torch.uint32 + topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt( + gating_output=gating_output[~is_padding], + topk=6, + renormalize=True, + routed_scaling_factor=1.5, + e_score_correction_bias=correction_bias, + ) + # uint32 CUDA tensors do not support boolean-mask indexing; widen first. + torch.testing.assert_close( + topk_ids.to(torch.int64)[~is_padding], + topk_ids_ref.to(torch.int64), + atol=0, + rtol=0, + ) + torch.testing.assert_close( + topk_weights[~is_padding], topk_weights_ref, atol=2e-5, rtol=2e-5 + ) + + +@pytest.mark.skipif( + not current_platform.is_cuda(), + reason="The DeepSeek V4 fast path is CUDA-only.", +) +def test_dsv4_fast_topk_padding(monkeypatch: pytest.MonkeyPatch): + """Verify the DSV4 fast path removes graph-padding rows from routing.""" + torch.manual_seed(0) + num_tokens = 17 + num_experts = 256 + hidden_states = torch.randn((num_tokens, 64), dtype=torch.float32, device="cuda") + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.float32, device="cuda" + ) + correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda") + is_padding = torch.zeros(num_tokens, dtype=torch.bool, device="cuda") + is_padding[1::2] = True + gating_output[is_padding] = float("nan") + + monkeypatch.setattr( + "vllm.model_executor.layers.fused_moe.router." + "fused_topk_bias_router._get_padding_mask", + lambda _: is_padding, + ) + topk_weights, topk_ids = fused_topk_bias( + hidden_states=hidden_states, + gating_output=gating_output, + scoring_func="sqrtsoftplus", + e_score_correction_bias=correction_bias, + topk=6, + renormalize=True, + routed_scaling_factor=1.5, + ) + + assert torch.equal(topk_ids[is_padding], torch.full_like(topk_ids[is_padding], -1)) + assert torch.equal( + topk_weights[is_padding], torch.zeros_like(topk_weights[is_padding]) + ) + + topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt( + gating_output=gating_output[~is_padding], + topk=6, + renormalize=True, + routed_scaling_factor=1.5, + e_score_correction_bias=correction_bias, + ) + torch.testing.assert_close(topk_ids[~is_padding], topk_ids_ref, atol=0, rtol=0) + torch.testing.assert_close( + topk_weights[~is_padding], topk_weights_ref, atol=2e-5, rtol=2e-5 + ) + + # The mask buffer is persistent under CUDA graph replay, but its contents + # change with every batch. Verify that the kernel reads those contents at + # runtime rather than specializing on the mask captured above. + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_weights, graph_ids = dsv4_topk( + gating_output, + correction_bias, + torch.int32, + 1.5, + is_padding=is_padding, + ) + + is_padding.logical_not_() + graph.replay() + assert torch.equal( + graph_ids[is_padding], torch.full_like(graph_ids[is_padding], -1) + ) + assert torch.equal( + graph_weights[is_padding], torch.zeros_like(graph_weights[is_padding]) + ) + + @pytest.mark.skipif( not current_platform.is_cuda_alike(), reason="This test is skipped on non-CUDA platform.", diff --git a/tests/models/kimi_k3/test_attn_res.py b/tests/models/kimi_k3/test_attn_res.py index 574f7a5925..366abe0221 100644 --- a/tests/models/kimi_k3/test_attn_res.py +++ b/tests/models/kimi_k3/test_attn_res.py @@ -17,6 +17,26 @@ EPS = 1e-5 attn_res_module = importlib.import_module("vllm.models.kimi_k3.nvidia.ops.attn_res") +def _on_rocm_below_10() -> bool: + if not current_platform.is_rocm(): + return False + from vllm.platforms.rocm import get_rocm_version + + return (get_rocm_version() or (0,)) < (10,) + + +# The Triton bundled with ROCm < 10 (3.7.x) crashes in the AMD +# CanonicalizePointers pass on the kernel's tl.where over pointer tensors. Only +# the num_blocks > 0 loop contains it. ROCm serves AttnRes from +# vllm/models/kimi_k3/amd/ops/attn_res.py instead, so nothing real is lost. +_OLD_ROCM = _on_rocm_below_10() + + +def _skip_on_old_rocm(num_blocks: int) -> None: + if _OLD_ROCM and num_blocks > 0: + pytest.skip("Triton on ROCm < 10 cannot compile this kernel's pointer select") + + def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor: storage = torch.randn( *shape[:-1], @@ -79,6 +99,7 @@ def test_attn_res( ): if backend == "nvidia" and not current_platform.is_device_capability_family(100): pytest.skip("NVIDIA AttnRes requires the SM100 family") + _skip_on_old_rocm(num_blocks) prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding) delta = ( @@ -133,6 +154,7 @@ def test_attn_res( @pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1)) def test_attn_res_block_counts(num_blocks: int): + _skip_on_old_rocm(num_blocks) prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) blocks = torch.randn( 1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16 @@ -169,6 +191,7 @@ def test_attn_res_block_counts(num_blocks: int): def test_attn_res_without_output_norm(): + _skip_on_old_rocm(MAX_BLOCKS) prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) delta = torch.randn_like(prefix) blocks = torch.randn( diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 39481456fd..f2f8ed6bd6 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -855,7 +855,7 @@ def test_quant_method_dispatch_ignored(default_vllm_config): UnquantizedFusedMoEMethod, ) - dynamic_mxfp4_config = _make_qtensor_config( + mxfp4_config = _make_qtensor_config( { "dtype": "fp4", "qscheme": "per_group", @@ -866,24 +866,23 @@ def test_quant_method_dispatch_ignored(default_vllm_config): None, exclude=["self_attn.q_proj", "mlp.down_proj"], ) - dynamic_mxfp4_config.dynamic_mxfp4_quant = True - - assert dynamic_mxfp4_config.get_quant_method_target( - "self_attn.q_proj", LinearBase - ) == (kMxfp4Static, None, QuarkLinearMethod) - attention_proj = TestLinear() - assert isinstance( - dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"), - QuarkLinearMethod, + assert mxfp4_config.get_quant_method_target("self_attn.q_proj", LinearBase) == ( + None, + None, + UnquantizedLinearMethod, ) - assert isinstance(attention_proj.scheme, QuarkOCP_MX) - assert attention_proj.scheme.dynamic_mxfp4_quant - - assert dynamic_mxfp4_config.get_quant_method_target( - "mlp.down_proj", LinearBase - ) == (None, None, UnquantizedLinearMethod) assert isinstance( - dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"), + mxfp4_config.get_quant_method(TestLinear(), "self_attn.q_proj"), + UnquantizedLinearMethod, + ) + + assert mxfp4_config.get_quant_method_target("mlp.down_proj", LinearBase) == ( + None, + None, + UnquantizedLinearMethod, + ) + assert isinstance( + mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"), UnquantizedLinearMethod, ) diff --git a/tests/test_config.py b/tests/test_config.py index 02639db26e..0b8a10504c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,6 +12,7 @@ from unittest.mock import patch import pydantic import pytest +import torch from huggingface_hub import ResolvedRevision from pydantic import ValidationError @@ -37,7 +38,7 @@ from vllm.config import ( WatermarkConfig, update_config, ) -from vllm.config.compilation import CompilationMode, CUDAGraphMode +from vllm.config.compilation import CompilationMode, CUDAGraphMode, PassConfig from vllm.config.kernel import IrOpPriorityConfig from vllm.config.load import LoadConfig from vllm.config.mamba import MambaBackendEnum @@ -656,6 +657,70 @@ def test_breakable_cudagraph_platform_default( default_breakable_cudagraph_architectures.cache_clear() +@pytest.mark.parametrize( + "case,hidden,heads,intermediate,tp,expected", + [ + ("bf16", 2048, (16, 8), 6144, 1, True), + ("bi-off", 2048, (16, 8), 6144, 1, False), + ("opt-out", 2048, (16, 8), 6144, 1, False), + ("eager", 2048, (16, 8), 6144, 1, False), + ("sp", 2048, (16, 8), 6144, 1, False), + ("fp16", 2048, (16, 8), 6144, 1, False), + ("quantized", 2048, (16, 8), 6144, 1, False), + ("untuned", 4096, (32, 32), 11008, 1, False), + ("tp4-tuned", 4096, (8, 2), 12288, 4, True), + ("list-intermediate", 2048, (16, 8), [2048, 4096], 1, False), + ("no-table", 2048, None, 6144, 1, False), + ], +) +def test_batch_invariant_breakable_cudagraph( + monkeypatch, case, hidden, heads, intermediate, tp, expected +): + from vllm.config.vllm import default_breakable_cudagraph_architectures + from vllm.model_executor.determinism import batch_invariant_configs as bi_configs + + monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0" if case == "bi-off" else "1") + monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) + if case == "opt-out": + monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0") + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + monkeypatch.setattr(current_platform, "is_rocm", lambda: False) + table = None if case == "no-table" else {(12288, 2048): None, (6144, 4096): None} + monkeypatch.setattr(current_platform, "get_device_capability", lambda: None) + monkeypatch.setattr(bi_configs, "_get_tuned_matmul_arch_family", lambda cap: "test") + monkeypatch.setattr( + bi_configs, + "_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS", + {"test": table} if table else {}, + ) + default_breakable_cudagraph_architectures.cache_clear() + config = object.__new__(VllmConfig) + config.model_config = SimpleNamespace( + architectures=["Qwen3ForCausalLM"], + enforce_eager=case == "eager", + dtype=torch.float16 if case == "fp16" else torch.bfloat16, + quantization="fp8" if case == "quantized" else None, + hf_text_config=SimpleNamespace(intermediate_size=intermediate), + get_hidden_size=lambda: hidden, + get_head_size=lambda: 128, + get_num_attention_heads=lambda pc: heads[0], + get_num_kv_heads=lambda pc: heads[1], + ) + config.parallel_config = SimpleNamespace(tensor_parallel_size=tp) + config.compilation_config = ( + CompilationConfig(pass_config=PassConfig(enable_sp=True)) + if case == "sp" + else CompilationConfig() + ) + try: + assert config._maybe_enable_breakable_cudagraph() is expected + if expected: + assert config.compilation_config.mode == CompilationMode.NONE + finally: + os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) + default_breakable_cudagraph_architectures.cache_clear() + + @pytest.mark.parametrize( ("model_type", "expected_architecture"), [ diff --git a/tests/v1/kv_connector/mooncake_integration/test_accuracy_rocm.py b/tests/v1/kv_connector/mooncake_integration/test_accuracy_rocm.py new file mode 100644 index 0000000000..766c4adabe --- /dev/null +++ b/tests/v1/kv_connector/mooncake_integration/test_accuracy_rocm.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Mooncake P/D accuracy with same-host HIP transport on ROCm.""" + +import pytest + +from tests.utils import spawn_new_process_for_each_test +from tests.v1.kv_connector.rocm_pd_accuracy_utils import ( + QWEN_MODEL, + run_rocm_pd_accuracy, +) +from vllm.platforms import current_platform + +pytestmark = [ + pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"), + pytest.mark.distributed(num_gpus=4), +] + + +@pytest.mark.parametrize( + "model,prefill_tp,decode_tp", + [ + pytest.param(QWEN_MODEL, 1, 1, id="mooncake-hip-qwen-p1-d1"), + pytest.param(QWEN_MODEL, 2, 2, id="mooncake-hip-qwen-p2-d2"), + ], +) +@spawn_new_process_for_each_test +def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch): + run_rocm_pd_accuracy("mooncake", model, prefill_tp, decode_tp, monkeypatch) diff --git a/tests/v1/kv_connector/nixl_integration/__init__.py b/tests/v1/kv_connector/nixl_integration/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/kv_connector/nixl_integration/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/kv_connector/nixl_integration/test_accuracy_rocm.py b/tests/v1/kv_connector/nixl_integration/test_accuracy_rocm.py new file mode 100644 index 0000000000..5f83330ef9 --- /dev/null +++ b/tests/v1/kv_connector/nixl_integration/test_accuracy_rocm.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NIXL P/D accuracy with AITER unified attention on ROCm.""" + +import pytest + +from tests.utils import spawn_new_process_for_each_test +from tests.v1.kv_connector.rocm_pd_accuracy_utils import ( + DEEPSEEK_MODEL, + QWEN_MODEL, + run_rocm_pd_accuracy, +) +from vllm.platforms import current_platform + +pytestmark = [ + pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"), + pytest.mark.distributed(num_gpus=4), +] + + +@pytest.mark.parametrize( + "model,prefill_tp,decode_tp", + [ + pytest.param(QWEN_MODEL, 2, 2, id="nixl-aiter-qwen-p2-d2"), + pytest.param(QWEN_MODEL, 1, 2, id="nixl-aiter-qwen-p1-d2"), + pytest.param(QWEN_MODEL, 2, 1, id="nixl-aiter-qwen-p2-d1"), + pytest.param(DEEPSEEK_MODEL, 1, 1, id="nixl-aiter-deepseek-p1-d1"), + pytest.param(DEEPSEEK_MODEL, 1, 2, id="nixl-aiter-deepseek-p1-d2"), + pytest.param(DEEPSEEK_MODEL, 2, 1, id="nixl-aiter-deepseek-p2-d1"), + ], +) +@spawn_new_process_for_each_test +def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch): + run_rocm_pd_accuracy("nixl", model, prefill_tp, decode_tp, monkeypatch) diff --git a/tests/v1/kv_connector/rocm_pd_accuracy_utils.py b/tests/v1/kv_connector/rocm_pd_accuracy_utils.py new file mode 100644 index 0000000000..e515c0a0bd --- /dev/null +++ b/tests/v1/kv_connector/rocm_pd_accuracy_utils.py @@ -0,0 +1,254 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared ROCm P/D accuracy harness for Mooncake and NIXL integration tests. + +Keep the models, TP pairs, full GSM8K workload, and acceptance checks of the +Mooncake and FlashInfer/NIXL sweeps. Mooncake uses same-host HIP IPC; NIXL uses +AITER unified attention. These cases do not exercise RDMA or FlashInfer. +""" + +import contextlib +import importlib +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import requests +from prometheus_client.parser import text_string_to_metric_families + +from tests.utils import RemoteOpenAIServer +from vllm.platforms import current_platform +from vllm.utils.network_utils import get_open_port + +_ROOT = Path(__file__).resolve().parents[3] +QWEN_MODEL = "Qwen/Qwen3-0.6B" +DEEPSEEK_MODEL = "deepseek-ai/deepseek-vl2-tiny" +_REVISIONS = { + QWEN_MODEL: "c1899de289a04d12100db370d81485cdf75e47ca", + DEEPSEEK_MODEL: "66c54660eae7e90c9ba259bfdf92d07d6e3ce8aa", +} + + +def _metric_total(server, name: str, **labels: str) -> float: + response = requests.get(server.url_for("metrics"), timeout=10) + response.raise_for_status() + return sum( + sample.value + for family in text_string_to_metric_families(response.text) + for sample in family.samples + if sample.name == name + and all(sample.labels.get(key) == value for key, value in labels.items()) + ) + + +@contextlib.contextmanager +def _proxy(connector: str, prefill, decode, bootstrap_port: int): + port = get_open_port() + if connector == "mooncake": + script = ( + _ROOT / "examples/disaggregated/mooncake_connector/" + "mooncake_connector_proxy.py" + ) + args = [ + "--prefill", + prefill.url_root, + str(bootstrap_port), + "--decode", + decode.url_root, + ] + else: + script = Path(__file__).parent / "nixl_integration/toy_proxy_server.py" + args = [ + "--prefiller-hosts", + "127.0.0.1", + "--prefiller-ports", + str(prefill.port), + "--decoder-hosts", + "127.0.0.1", + "--decoder-ports", + str(decode.port), + ] + process = subprocess.Popen( + [sys.executable, str(script), "--port", str(port), *args], + start_new_session=True, + ) + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + assert process.poll() is None, "P/D proxy exited during startup" + try: + requests.get(f"http://127.0.0.1:{port}/", timeout=1) + break + except requests.ConnectionError: + time.sleep(0.2) + else: + raise TimeoutError("P/D proxy did not start") + yield f"http://127.0.0.1:{port}/v1" + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + + +def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch): + """Transferred KV must feed decoding while full upstream accuracy holds.""" + # Preserve the caller's allocation instead of querying host-wide SMI indices. + selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES") + selector = next((key for key in selectors if key in os.environ), selectors[0]) + visible = os.getenv(selector) + devices = ( + visible.split(",") + if visible is not None + else [str(i) for i in range(current_platform.device_count())] + ) + assert len(devices) >= prefill_tp + decode_tp, "Insufficient allocated GPUs" + for key in selectors: + if key != selector: + monkeypatch.delenv(key, raising=False) + + if connector == "mooncake": + import mooncake.engine + + assert mooncake.engine.SUPPORT_HIP, "Mooncake requires its ROCm/HIP build" + connector_name = "MooncakeConnector" + extra = {"mooncake_protocol": "hip", "device_name": ""} + else: + from vllm.distributed.nixl_utils import NixlWrapper + + assert NixlWrapper is not None, "NIXL ROCm bindings must import" + connector_name = "NixlConnector" + extra = {} + + common_args = [ + "--revision", + _REVISIONS[model], + "--dtype", + "bfloat16", + "--enforce-eager", + "--distributed-executor-backend", + "mp", + "--block-size", + "128", + "--max-num-seqs", + "128", + "--max-num-batched-tokens", + "4096", + "--gpu-memory-utilization", + "0.2", + "--kv-cache-memory-bytes", + str(8 * 1024**3), + "--moe-backend", + "triton", + "--attention-backend", + "ROCM_AITER_UNIFIED_ATTN", + ] + common_env = { + "PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"], + "VLLM_HOST_IP": "127.0.0.1", + "VLLM_WORKER_MULTIPROC_METHOD": "spawn", + "VLLM_USE_V2_MODEL_RUNNER": "1", + "VLLM_ROCM_USE_AITER": "1", + "VLLM_KV_CACHE_LAYOUT": "HND", + } + bootstrap_port = get_open_port() + with contextlib.ExitStack() as stack: + servers: list[RemoteOpenAIServer] = [] + stack.callback(RemoteOpenAIServer.shutdown_many, servers) + for role, tp, assigned in ( + ("kv_producer", prefill_tp, devices[:prefill_tp]), + ("kv_consumer", decode_tp, devices[prefill_tp : prefill_tp + decode_tp]), + ): + config = { + "kv_connector": connector_name, + "kv_role": role, + "kv_connector_extra_config": extra, + } + env = { + **common_env, + selector: ",".join(assigned), + "VLLM_PORT": str(get_open_port()), + "VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()), + "VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port), + } + servers.append( + RemoteOpenAIServer( + model, + [ + *common_args, + "--tensor-parallel-size", + str(tp), + "--kv-transfer-config", + json.dumps(config), + ], + env_dict=env, + max_wait_seconds=1200, + ) + ) + prefill, decode = servers + proxy_url = stack.enter_context( + _proxy(connector, prefill, decode, bootstrap_port) + ) + + # Reuse the full upstream evaluation and its model-specific assertions. + accuracy = importlib.import_module( + f"tests.v1.kv_connector.{connector}_integration.test_accuracy" + ) + monkeypatch.setattr(accuracy, "MODEL_NAME", model) + monkeypatch.setattr(accuracy, "BASE_URL", proxy_url) + assert model in accuracy.EXPECTED_VALUES + + def transfer_totals(): + totals = { + "external_tokens": _metric_total( + decode, + "vllm:prompt_tokens_by_source_total", + source="external_kv_transfer", + ) + } + if connector == "nixl": + totals["nixl_bytes"] = _metric_total( + decode, "vllm:nixl_bytes_transferred_sum" + ) + return totals + + simple_evaluate = accuracy.lm_eval.simple_evaluate + + def evaluate_with_transfer_checks(*args, **kwargs): + # The upstream helper runs a smoke prompt before GSM8K. Wait until + # its counters appear, then require new transfers during evaluation. + deadline = time.monotonic() + 15 + before = transfer_totals() + while not all(value > 0 for value in before.values()): + assert time.monotonic() < deadline, ( + f"Smoke prompt did not transfer KV: {before}" + ) + time.sleep(1) + before = transfer_totals() + + # The API adapter loads a local tokenizer because these proxies do + # not expose /tokenize. Match it to the server's pinned checkpoint. + kwargs["model_args"] += f",revision={_REVISIONS[model]}" + results = simple_evaluate(*args, **kwargs) + deadline = time.monotonic() + 15 + after = transfer_totals() + while not all(after[key] > value for key, value in before.items()): + assert time.monotonic() < deadline, ( + f"GSM8K did not transfer KV: before={before}, after={after}" + ) + time.sleep(1) + after = transfer_totals() + print(f"GSM8K transfer counters: before={before}, after={after}") + return results + + monkeypatch.setattr( + accuracy.lm_eval, "simple_evaluate", evaluate_with_transfer_checks + ) + accuracy.test_accuracy() diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index e35ead0015..a81c1571a1 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -50,7 +50,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( NixlKVConnectorStats, ) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( + ReadSpec, + TPMapping, +) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( get_base_request_id, ) @@ -344,6 +347,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): ) w._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list) + w._send_failures = set() w._sending_transfers_lock = threading.Lock() w._push_finished_blocks = {} w._pending_d_registrations = {} @@ -941,9 +945,10 @@ class TestPushWriterNotifs: assert request_id not in w._sending_transfers @pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)]) - def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma): - """P-side sends are completed by consumer notifs / lease expiry, - not by local WRITE-handle completion.""" + def test_completed_push_send_releases_blocks_without_consumer_notif( + self, pp_size, is_hma + ): + """All WRITEs completing lets P reclaim its blocks without a D ACK.""" w = self._pollable_worker() w.pp_size = pp_size w._is_hma_required = is_hma @@ -951,20 +956,14 @@ class TestPushWriterNotifs: w.nixl_wrapper.check_xfer_state.side_effect = ["DONE", "PROC", "DONE"] results = w.get_transfer_results() + assert request_id not in results.finished_sending + assert request_id in w._reqs_to_send assert w._sending_transfers[request_id] == [102] - assert request_id in w._reqs_to_send + assert w._evict_finished_inbox.empty() results = w.get_transfer_results() - - assert request_id not in results.finished_sending - assert request_id in w._reqs_to_send - assert request_id not in w._sending_transfers - - # The consumer notif is what completes the send. - w._pending_completion_notifs.put(f"{request_id}:1".encode()) - results = w.get_transfer_results() - assert request_id in results.finished_sending + assert results.finished_sending == {request_id} assert request_id not in w._reqs_to_send assert request_id not in w._reqs_to_process assert w._evict_finished_inbox.get_nowait() == request_id @@ -973,6 +972,91 @@ class TestPushWriterNotifs: assert w.nixl_wrapper.release_xfer_handle.call_count == 2 w.xfer_stats.record_kv_expired_req.assert_not_called() + @pytest.mark.parametrize("sibling_state", ["ERR", "DONE"]) + def test_push_failure_survives_until_lease_expiry(self, sibling_state): + """A later sibling completion must not turn a failed send into success.""" + w = self._pollable_worker() + request_id = self._make_sending_req(w) + w.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state] + + assert w.get_transfer_results().finished_sending == set() + assert w._sending_transfers[request_id] == [102] + assert w.get_transfer_results().finished_sending == set() + assert request_id not in w._sending_transfers + assert request_id in w._reqs_to_send + assert w._evict_finished_inbox.empty() + + w.expected_consumer_notifications_by_req = {} + w._reqs_to_send[request_id] = time.perf_counter() - 1 + assert w.get_transfer_results().finished_sending == {request_id} + assert request_id not in w._send_failures + assert w._evict_finished_inbox.get_nowait() == request_id + + @pytest.mark.parametrize("release_fails", [False, True]) + def test_push_submission_failure_is_remembered_after_sibling_success( + self, release_fails + ): + """A submission error remains a failure even if later polls succeed.""" + w = self._pollable_worker() + request_id = self._make_sending_req(w) + w.transfer_topo.block_size_ratio.return_value = 1 + w._apply_prefix_caching = MagicMock(return_value=([[1]], [[1]])) + w._compute_desc_ids = MagicMock(return_value=[0]) + w.dst_num_blocks = {w.engine_id: 8} + w.dst_region_num_blocks[w.engine_id] = [8] + w.dst_region_group_ids[w.engine_id] = [0] + w.dst_uses_region_group_mapping[w.engine_id] = False + w.nixl_wrapper.make_prepped_xfer.return_value = 101 + w.nixl_wrapper.transfer.side_effect = RuntimeError("submit") + if release_fails: + w.nixl_wrapper.release_xfer_handle.side_effect = [ + RuntimeError("release"), + None, + None, + ] + + handle = w._xfer_blocks( + read_spec=ReadSpec(0, [[1]], [[1]]), + dst_engine_id=w.engine_id, + request_id=request_id, + remote_request_id="decode-request", + local_xfer_side_handle=1, + remote_xfer_side_handle=2, + ) + assert handle == (101 if release_fails else None) + w._sending_transfers[request_id] = [102] + if handle is not None: + w._sending_transfers[request_id].append(handle) + w.nixl_wrapper.check_xfer_state.return_value = "DONE" + + assert w.get_transfer_results().finished_sending == set() + assert request_id in w._reqs_to_send + assert request_id not in w._sending_transfers + assert w._evict_finished_inbox.empty() + + @pytest.mark.parametrize("first_state", ["ERR", "DONE"]) + def test_push_completion_after_expiry_is_not_reported_twice(self, first_state): + """A WRITE finishing after lease expiry must not complete the request again.""" + w = self._pollable_worker() + request_id = self._make_sending_req(w) + w.nixl_wrapper.check_xfer_state.side_effect = [ + first_state, + "PROC", + "PROC", + "DONE", + ] + assert w.get_transfer_results().finished_sending == set() + + w.expected_consumer_notifications_by_req = {} + w._reqs_to_send[request_id] = time.perf_counter() - 1 + assert w.get_transfer_results().finished_sending == {request_id} + assert w._evict_finished_inbox.get_nowait() == request_id + + assert w.get_transfer_results().finished_sending == set() + assert request_id not in w._sending_transfers + assert request_id not in w._send_failures + assert w._evict_finished_inbox.empty() + # ----------------------------------------------------------------- # # Negative / error-path tests # diff --git a/tests/v1/spec_decode/test_dflash_prepare_inputs.py b/tests/v1/spec_decode/test_dflash_prepare_inputs.py index 16d6d8e516..f3f94e1a39 100644 --- a/tests/v1/spec_decode/test_dflash_prepare_inputs.py +++ b/tests/v1/spec_decode/test_dflash_prepare_inputs.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from contextlib import contextmanager from types import SimpleNamespace import numpy as np @@ -8,7 +9,9 @@ import pytest import torch from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.worker.gpu.spec_decode.dflash import speculator as dflash_speculator from vllm.v1.worker.gpu.spec_decode.dflash.speculator import ( + DFlashSpeculator, prepare_dflash_inputs, ) @@ -33,6 +36,7 @@ def _run_prepare( input_buffers = SimpleNamespace( input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device), positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device), + is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device), query_start_loc=torch.full( (max_num_reqs + 1,), -1, dtype=torch.int32, device=device ), @@ -140,6 +144,114 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix(): assert out.temperature[2].item() == 1.0 assert out.seeds[2].item() == 17 + assert not out.input_buffers.is_padding[:3].any() + assert out.input_buffers.is_padding[3:].all() + assert out.input_buffers.input_ids[3:].cpu().tolist() == [0] * 13 + assert out.input_buffers.positions[3:].cpu().tolist() == [0] * 13 + + +def test_prepare_dflash_inputs_compacts_noncontiguous_request_slots(): + device = torch.device("cuda") + max_num_reqs = 4 + max_num_tokens = 16 + num_speculative_steps = 3 + input_buffers = SimpleNamespace( + input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device), + positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device), + is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device), + query_start_loc=torch.full( + (max_num_reqs + 1,), -1, dtype=torch.int32, device=device + ), + seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device), + ) + input_batch = SimpleNamespace( + num_reqs=2, + num_scheduled_tokens=np.array([4, 4], dtype=np.int32), + positions=torch.tensor( + [10, 11, 12, 13, 20, 21, 22, 23], + dtype=torch.int64, + device=device, + ), + query_start_loc=torch.tensor([0, 4, 8], dtype=torch.int32, device=device), + # Active batch rows are compact, while request state remains in slots 3 and 1. + idx_mapping=torch.tensor([3, 1], dtype=torch.int32, device=device), + ) + query_slot_mapping = torch.full( + (max_num_tokens,), -2, dtype=torch.int64, device=device + ) + context_positions = torch.full( + (max_num_tokens,), -1, dtype=torch.int64, device=device + ) + context_slot_mapping = torch.full( + (max_num_tokens,), -2, dtype=torch.int64, device=device + ) + sample_indices = torch.full( + (max_num_reqs * num_speculative_steps,), + -1, + dtype=torch.int64, + device=device, + ) + sample_pos = torch.full_like(sample_indices, -1) + sample_idx_mapping = torch.full( + sample_indices.shape, -1, dtype=torch.int32, device=device + ) + temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device) + seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device) + input_temperature = torch.tensor( + [0.0, 0.5, 0.0, 1.0], dtype=torch.float32, device=device + ) + input_seeds = torch.tensor([0, 11, 0, 33], dtype=torch.int64, device=device) + last_sampled = torch.tensor([0, 77, 0, 99], dtype=torch.int64, device=device) + next_prefill_tokens = torch.zeros_like(last_sampled) + block_table = torch.tensor( + [[0, 0, 7, 8, 9, 10, 11, 12], [0, 0, 13, 14, 15, 16, 17, 18]], + dtype=torch.int32, + device=device, + ) + + prepare_dflash_inputs( + input_buffers, + query_slot_mapping, + context_positions, + context_slot_mapping, + sample_indices, + sample_pos, + sample_idx_mapping, + temperature, + seeds, + input_batch, + torch.tensor([1, 1], dtype=torch.int32, device=device), + torch.tensor([2, 1], dtype=torch.int32, device=device), + last_sampled, + next_prefill_tokens, + input_temperature, + input_seeds, + block_table, + 4, + 0, + 1, + 1, + 123, + num_speculative_steps, + num_speculative_steps, + max_num_reqs, + max_num_tokens, + 128, + sample_from_anchor=True, + ) + torch.accelerator.synchronize() + + # Query rows follow compact batch order, but every persistent state lookup + # follows idx_mapping instead of accidentally using the compact row index. + assert input_buffers.input_ids[:6].cpu().tolist() == [99, 123, 123, 77, 123, 123] + assert input_buffers.positions[:6].cpu().tolist() == [12, 13, 14, 23, 24, 25] + assert sample_indices[:6].cpu().tolist() == [0, 1, 2, 3, 4, 5] + assert sample_idx_mapping[:6].cpu().tolist() == [3, 3, 3, 1, 1, 1] + assert temperature.cpu().tolist() == [0.0, 0.5, 0.0, 1.0] + assert seeds.cpu().tolist() == [0, 11, 0, 33] + assert not input_buffers.is_padding[:6].any() + assert input_buffers.is_padding[6:].all() + def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp(): out = _run_prepare( @@ -174,3 +286,40 @@ def test_prepare_dflash_inputs_never_writes_the_null_block(): PAD_SLOT_ID, PAD_SLOT_ID, ] + + +def test_dflash_forward_context_receives_draft_padding_mask(monkeypatch): + device = torch.device("cuda") + input_buffers = SimpleNamespace( + input_ids=torch.tensor([11, 12, 0, 0], dtype=torch.int32, device=device), + positions=torch.tensor([7, 8, 0, 0], dtype=torch.int64, device=device), + is_padding=torch.tensor([False, False, True, True], device=device), + ) + observed = None + + @contextmanager + def fake_set_forward_context(*args, **kwargs): + nonlocal observed + observed = kwargs["is_padding"].clone() + yield + + monkeypatch.setattr( + dflash_speculator, "set_forward_context", fake_set_forward_context + ) + speculator = SimpleNamespace( + input_buffers=input_buffers, + vllm_config=SimpleNamespace(), + model=lambda **kwargs: kwargs["input_ids"], + ) + + result = DFlashSpeculator._run_model( + speculator, + num_tokens=4, + attn_metadata=None, + slot_mappings=None, + num_tokens_across_dp=None, + ) + + assert result.tolist() == [11, 12, 0, 0] + assert observed is not None + assert observed.tolist() == [False, False, True, True] diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 32341ce944..e77d278116 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -31,6 +31,7 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange from vllm.platforms import current_platform +from vllm.platforms.interface import Platform from vllm.sampling_params import SamplingParams from vllm.utils.mem_constants import GiB_bytes from vllm.utils.system_utils import update_environment_variables @@ -425,6 +426,62 @@ def test_select_common_block_size_no_valid_option(): select_common_block_size(48, [backend_a, backend_b]) +def _mock_backend(supported: list, *, exact: bool = False): + """Backend accepting multiples of ``supported``, or only those exact sizes + when ``exact`` (as CPU_MLA does).""" + from vllm.v1.attention.backend import AttentionBackend + + class _MockBackendCls(AttentionBackend): + @staticmethod + def get_name() -> str: + return "MOCK_EXACT" if exact else "MOCK" + + @staticmethod + def get_supported_kernel_block_sizes(): + return list(supported) + + if exact: + + @classmethod + def supports_block_size(cls, block_size: int | None) -> bool: + return block_size is None or block_size in supported + + return _MockBackendCls + + +@pytest.mark.parametrize( + "backends,expected", + [ + # A lone backend keeps the default; MultipleOf(8) makes a + # minimum-based regression return 8 instead of 16. + ([[MultipleOf(8)]], 16), + # Sparse-MLA main backend (32 or 64) beside an indexer supporting only + # 64: the main backend alone picks 32, which the indexer rejects at + # select_common_block_size time. + ([[32, 64], [64]], 64), + ([[32, 64]], 32), + # Neither divides the other, so only the LCM satisfies both. + ([[32], [48]], 96), + ], +) +def test_preferred_block_size_satisfies_every_backend(backends, expected): + classes = [_mock_backend(s) for s in backends] + assert Platform._preferred_block_size_for_backends(classes, 16, None) == expected + + +def test_preferred_block_size_searches_past_an_exact_size_backend(): + # Extending greedily picks lcm(16, 32) = 32, which the exact backend + # rejects; 96 is accepted by both. + classes = [_mock_backend([16, 96], exact=True), _mock_backend([MultipleOf(32)])] + assert Platform._preferred_block_size_for_backends(classes, 16, None) == 96 + + +def test_preferred_block_size_rejects_backends_with_no_common_size(): + classes = [_mock_backend([16], exact=True), _mock_backend([MultipleOf(64)])] + with pytest.raises(ValueError, match="share no supported KV cache block size"): + Platform._preferred_block_size_for_backends(classes, 16, None) + + def test_set_active_mm_loras_builds_tower_and_connector_mappings(): model = Mock() model.get_mm_lora_token_counts.side_effect = ( diff --git a/tests/v1/worker/test_pp_utils.py b/tests/v1/worker/test_pp_utils.py index cfc92479f5..8dfdf2e6d4 100644 --- a/tests/v1/worker/test_pp_utils.py +++ b/tests/v1/worker/test_pp_utils.py @@ -5,16 +5,35 @@ from unittest.mock import Mock import numpy as np +import pytest +import torch -from vllm.v1.worker.gpu import pp_utils +from vllm.v1.worker.gpu import model_runner, pp_utils -def _batch(num_computed, prefill_len, num_scheduled): +def _cuda_handler(max_sample_len=6): + handler = object.__new__(pp_utils.PPHandler) + handler.is_last_rank = True + handler.disabled = False + handler.max_sample_len = max_sample_len + handler.last_rank = 1 + handler.broadcast_group = Mock() + handler.device = torch.device("cuda") + handler.main_stream = torch.cuda.current_stream() + handler.broadcast_stream = torch.cuda.Stream() + return handler + + +def _batch(num_computed, prefill_len, num_scheduled, idx_mapping=None): + num_reqs = len(num_computed) + if idx_mapping is None: + idx_mapping = list(range(num_reqs)) return Mock( - num_reqs=len(num_computed), + num_reqs=num_reqs, num_computed_tokens_np=np.array(num_computed, dtype=np.int32), prefill_len_np=np.array(prefill_len, dtype=np.int32), num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32), + idx_mapping=torch.tensor(idx_mapping, dtype=torch.int64), ) @@ -81,3 +100,130 @@ def test_decode_row_ahead_of_a_prefill_chunk(): assert mask is not None assert mask.tolist() == [True, False] + + +def test_disabled_handler_skips_broadcast_and_receive(monkeypatch): + """While disabled (warmup), neither side enqueues a broadcast op.""" + sent = [] + monkeypatch.setattr( + pp_utils.torch.distributed, + "broadcast", + lambda *args, **kwargs: sent.append((args, kwargs)), + ) + + handler = object.__new__(pp_utils.PPHandler) + handler.set_disabled(True) + + handler.is_last_rank = False + assert handler.receive(Mock()) is False + + handler.is_last_rank = True + assert handler.broadcast(Mock(), Mock(), Mock(), Mock()) is None + + assert sent == [] + + handler.set_disabled(False) + assert handler.disabled is False + + +def test_alloc_combined_keeps_unbind_views_16_byte_aligned(): + """Triton specializes on pointer alignment: an unaligned `num_rejected` + would compile a second `_post_update_kernel` variant at serving time, + where the in-flight broadcast NCCL kernel can block the module load.""" + for num_reqs in range(1, 9): + combined = pp_utils._alloc_combined(num_reqs, torch.device("cpu")) + num_sampled, num_rejected = combined.unbind(dim=0) + assert num_sampled.data_ptr() % 16 == 0 + assert num_rejected.data_ptr() % 16 == 0 + assert combined.shape[1] >= num_reqs + + +def test_warmup_pp_decode_update_matches_serving_specialization(monkeypatch): + """The warmup launch must hit the same triton specialization as serving. + + A mismatch means the first real ``update_pp_decode_requests`` recompiles + mid-serving, where the in-flight broadcast NCCL kernel blocks the CUDA + module load and deadlocks the pipeline. + """ + calls = [] + monkeypatch.setattr(model_runner, "post_update", lambda *args: calls.append(args)) + + runner = object.__new__(model_runner.GPUModelRunner) + runner.device = torch.device("cpu") + runner.pp_handler = Mock(max_sample_len=3) + runner.req_states = Mock() + + runner.warmup_pp_decode_update() + + assert len(calls) == 1 + args = calls[0] + idx_mapping, _, _, output_bin_counts = args[:4] + sampled_tokens, num_sampled, num_rejected, query_start_loc = args[4:8] + broadcast_drafts, draft_tokens_out = args[10:12] + assert idx_mapping.tolist() == [-1] and idx_mapping.dtype == torch.int64 + assert output_bin_counts is None + assert query_start_loc is None + assert sampled_tokens.shape == (1, 3) and sampled_tokens.dtype == torch.int64 + assert num_sampled.dtype == torch.int32 + assert num_rejected.dtype == torch.int32 + # Spec-enabled PP handlers receive drafts over the broadcast; the warmup + # must compile that specialization (non-None draft pointers) too. + assert broadcast_drafts.shape == (1, 2) and broadcast_drafts.dtype == torch.int64 + assert draft_tokens_out is runner.req_states.draft_tokens + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA stream") +def test_broadcast_pads_plain_sampler_rows_to_max_sample_len(monkeypatch): + """The wire shape must not depend on whether the batch carried drafts: + the receiver always allocates [num_reqs, max_sample_len], and a NCCL + broadcast with mismatched counts hangs the receiver.""" + sent = [] + monkeypatch.setattr( + pp_utils.torch.distributed, + "broadcast", + lambda tensor, **kw: sent.append(tensor), + ) + handler = _cuda_handler() + batch = _batch(num_computed=[10], prefill_len=[8], num_scheduled=[1]) + + handler.broadcast( + torch.zeros(1, 1, dtype=torch.int64, device="cuda"), # plain sampler + torch.ones(1, dtype=torch.int32, device="cuda"), + torch.zeros(1, dtype=torch.int32, device="cuda"), + batch, + ) + + assert sent[0].shape == (1, 6) + assert sent[1].shape == (2, 4) + torch.accelerator.synchronize() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA stream") +def test_broadcast_drafts_gathers_fresh_rows_from_the_table(monkeypatch): + sent = [] + monkeypatch.setattr( + pp_utils.torch.distributed, + "broadcast", + lambda tensor, **kw: sent.append(tensor), + ) + handler = _cuda_handler() + # The batch's single row maps to request-state row 2. + batch = _batch(num_computed=[10], prefill_len=[8], num_scheduled=[1]) + batch.idx_mapping = torch.tensor([2], dtype=torch.int64, device="cuda") + table = torch.arange(20, dtype=torch.int64, device="cuda").view(4, 5) + + handler.broadcast_drafts(table, batch) + + assert sent[0].shape == (1, 5) + # The payload is a gather into a fresh tensor: propose() overwrites its + # persistent buffer on the next step, possibly before this send completes. + assert sent[0].data_ptr() != table.data_ptr() + assert sent[0].cpu().tolist() == [table[2].cpu().tolist()] + + # An all-prefill batch sends nothing (receive() enqueues nothing either). + sent.clear() + prefill_batch = _batch(num_computed=[0], prefill_len=[4096], num_scheduled=[448]) + prefill_batch.idx_mapping = prefill_batch.idx_mapping.cuda() + handler.broadcast_drafts(table, prefill_batch) + assert sent == [] + torch.accelerator.synchronize() diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index e5ce96cf24..0a28c06ee7 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -3362,6 +3362,115 @@ class rocm_aiter_ops: rotary_dim=kernel_rotary_dim, ) + @staticmethod + def fused_qk_norm_mrope_and_cache( + qkv: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + is_neox: bool, + mrope_section: list[int], + is_interleaved: bool, + rms_norm_eps: float, + q_out: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + slot_mapping: torch.Tensor, + k_scale: torch.Tensor, + v_scale: torch.Tensor, + block_size: int, + x: int, + rotary_dim: int = 0, + ) -> None: + from aiter.ops.fused_qk_norm_mrope_cache_quant import ( + fused_qk_norm_mrope_3d_cache_pts_quant_shuffle, + ) + + fused_qk_norm_mrope_3d_cache_pts_quant_shuffle( + qkv, + q_weight, + k_weight, + cos_sin_cache, + positions, + qkv.size(0), + num_heads_q, + num_heads_k, + num_heads_k, + head_dim, + is_neox, + mrope_section, + is_interleaved, + rms_norm_eps, + q_out, + k_cache, + v_cache, + slot_mapping, + k_scale, + v_scale, + None, + None, + False, + False, + block_size, + x, + rotary_dim, + ) + + @staticmethod + def do_qk_norm_mrope_kvcache_update( + qkv: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + is_neox: bool, + mrope_section: list[int], + is_interleaved: bool, + rms_norm_eps: float, + q_out: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + k_scale: torch.Tensor, + v_scale: torch.Tensor, + kv_cache_dtype: str, + rotary_dim: int = 0, + ) -> None: + if kv_cache_dtype.startswith("fp8"): + key_cache = key_cache.view(current_platform.fp8_dtype()) + value_cache = value_cache.view(current_platform.fp8_dtype()) + + rocm_aiter_ops.fused_qk_norm_mrope_and_cache( + qkv=qkv, + q_weight=q_weight, + k_weight=k_weight, + cos_sin_cache=cos_sin_cache, + positions=positions, + num_heads_q=num_heads_q, + num_heads_k=num_heads_k, + head_dim=head_dim, + is_neox=is_neox, + mrope_section=mrope_section, + is_interleaved=is_interleaved, + rms_norm_eps=rms_norm_eps, + q_out=q_out, + k_cache=key_cache, + v_cache=value_cache, + slot_mapping=slot_mapping, + k_scale=k_scale, + v_scale=v_scale, + block_size=key_cache.shape[1], + x=16 // key_cache.element_size(), + rotary_dim=rotary_dim, + ) + @staticmethod def triton_rope_and_cache( query: torch.Tensor, diff --git a/vllm/compilation/passes/fusion/matcher_utils.py b/vllm/compilation/passes/fusion/matcher_utils.py index 7fd44dd005..6d959c3067 100644 --- a/vllm/compilation/passes/fusion/matcher_utils.py +++ b/vllm/compilation/passes/fusion/matcher_utils.py @@ -31,6 +31,7 @@ from vllm.platforms import current_platform ROTARY_OP = torch.ops._C.rotary_embedding.default FLASHINFER_ROTARY_OP = torch.ops.vllm.flashinfer_rotary_embedding.default +MROPE_OP = torch.ops.vllm.mrope.default QUANT_OPS: dict[QuantKey, OpOverload] = { kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa: E501 @@ -162,6 +163,74 @@ class MatcherRotaryEmbedding(MatcherCustomOp): return result +class MatcherMRotaryEmbedding(MatcherCustomOp): + def __init__( + self, + is_neox: bool, + head_size: int, + rotary_dim: int, + num_heads: int, + num_kv_heads: int, + mrope_section: tuple[int, int, int], + mrope_interleaved: bool, + enabled: bool | None = None, + ) -> None: + if enabled is None: + enabled = RotaryEmbedding.enabled() + + super().__init__(enabled) + self.is_neox = is_neox + self.head_size = head_size + self.rotary_dim = rotary_dim + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads + self.q_size = self.num_heads * self.head_size + self.kv_size = self.num_kv_heads * self.head_size + self.mrope_section = mrope_section + self.mrope_interleaved = mrope_interleaved + + def inputs(self) -> list[torch.Tensor]: + positions = self.empty_int64(3, 5) + query = self.empty(5, self.q_size) + key = self.empty(5, self.kv_size) + cos_sin_cache = self.empty(4096, self.rotary_dim) + return [positions, query, key, cos_sin_cache] + + def forward_custom( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + cos_sin_cache: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + result = auto_functionalized( + MROPE_OP, + positions=positions, + query=query, + key=key, + cos_sin_cache=cos_sin_cache, + head_size=self.head_size, + rotary_dim=self.rotary_dim, + mrope_section_t=self.mrope_section[0], + mrope_section_h=self.mrope_section[1], + mrope_section_w=self.mrope_section[2], + mrope_interleaved=self.mrope_interleaved, + is_neox_style=self.is_neox, + ) + return result[1], result[2] + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + cos_sin_cache: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError( + "MRoPE fusion requires the rotary_embedding custom op" + ) + + class MatcherRMSNormGated(MatcherCustomOp): """Matches RMSNormGated with norm_before_gate=True and group_size=None.""" diff --git a/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py b/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py index 80eeb648f4..f418e8a7ef 100644 --- a/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py +++ b/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import inspect -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import ParamSpec import torch @@ -19,12 +19,19 @@ from vllm.model_executor.layers.attention.attention import ( Attention, get_attention_context, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape from vllm.platforms import current_platform -from vllm.utils.torch_utils import direct_register_custom_op +from vllm.utils.torch_utils import ( + _USE_LAYERNAME, + LayerNameType, + _encode_layer_name, + _resolve_layer_name, + direct_register_custom_op, +) from ..inductor_pass import enable_fake_mode from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass -from .matcher_utils import MatcherRotaryEmbedding +from .matcher_utils import MROPE_OP, MatcherMRotaryEmbedding, MatcherRotaryEmbedding from .rms_quant_fusion import empty_bf16, empty_fp32, empty_i64 logger = init_logger(__name__) @@ -51,8 +58,9 @@ def fused_qk_norm_rope_and_unified_kv_cache_update_impl( rms_norm_eps: float, cos_sin_cache: torch.Tensor, is_neox: bool, - layer_name: str = "", + layer_name: LayerNameType, ) -> torch.Tensor: + layer_name = _resolve_layer_name(layer_name) _, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name) if layer_slot_mapping is not None: attn_layer.impl.do_qk_norm_rope_kvcache_update( @@ -87,7 +95,7 @@ def fused_qk_norm_rope_and_unified_kv_cache_update_fake( rms_norm_eps: float, cos_sin_cache: torch.Tensor, is_neox: bool, - layer_name: str = "", + layer_name: LayerNameType, ) -> torch.Tensor: return torch.empty(0, device=qkv.device, dtype=qkv.dtype) @@ -100,6 +108,76 @@ direct_register_custom_op( ) +def fused_qk_norm_mrope_and_unified_kv_cache_update_impl( + q_out: torch.Tensor, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + mrope_section_t: int, + mrope_section_h: int, + mrope_section_w: int, + is_interleaved: bool, + rotary_dim: int, + layer_name: LayerNameType, +) -> torch.Tensor: + layer_name = _resolve_layer_name(layer_name) + _, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name) + if layer_slot_mapping is not None: + if positions.ndim == 1: + positions = positions.unsqueeze(0).expand(3, -1).contiguous() + attn_layer.impl.do_qk_norm_mrope_kvcache_update( + attn_layer, + qkv, + q_out, + positions, + q_weight, + k_weight, + rms_norm_eps, + cos_sin_cache, + is_neox, + (mrope_section_t, mrope_section_h, mrope_section_w), + is_interleaved, + rotary_dim, + kv_cache, + layer_slot_mapping, + ) + else: + q_out.zero_() + + return torch.empty(0, device=qkv.device, dtype=qkv.dtype) + + +def fused_qk_norm_mrope_and_unified_kv_cache_update_fake( + q_out: torch.Tensor, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + mrope_section_t: int, + mrope_section_h: int, + mrope_section_w: int, + is_interleaved: bool, + rotary_dim: int, + layer_name: LayerNameType, +) -> torch.Tensor: + return torch.empty(0, device=qkv.device, dtype=qkv.dtype) + + +direct_register_custom_op( + op_name="fused_qk_norm_mrope_and_unified_kv_cache_update", + op_func=fused_qk_norm_mrope_and_unified_kv_cache_update_impl, + mutates_args=["q_out"], + fake_impl=fused_qk_norm_mrope_and_unified_kv_cache_update_fake, +) + + # --------------------------------------------------------------------------- # Pattern: QK-norm + RoPE + unified_kv_cache_update # --------------------------------------------------------------------------- @@ -142,19 +220,27 @@ class QkNormRopeKvCachePattern: self.eps = eps self.is_neox = is_neox self.quant_query = quant_query + self.encoded_layer_name = _encode_layer_name(self.layer_name) + self.query_quant_group_shape = ( + layer.query_quant.group_shape + if layer.query_quant is not None + else GroupShape.PER_TENSOR + ) self.q_size = self.num_heads * self.head_size self.k_size = self.num_kv_heads * self.head_size self.v_size = self.num_kv_heads * self.head_size_v - self.rope_matcher = MatcherRotaryEmbedding( - is_neox=is_neox, - head_size=self.head_size, - num_heads=self.num_heads, - num_kv_heads=self.num_kv_heads, + self.rope_matcher: MatcherRotaryEmbedding | MatcherMRotaryEmbedding = ( + MatcherRotaryEmbedding( + is_neox=is_neox, + head_size=self.head_size, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + ) ) - def get_inputs(self) -> list[torch.Tensor]: + def get_inputs(self) -> list: T = 5 L = 4096 qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size) @@ -166,8 +252,13 @@ class QkNormRopeKvCachePattern: if self.quant_query: q_scale = empty_fp32(1) inputs += [q_scale] + if _USE_LAYERNAME: + inputs.append(self.encoded_layer_name) return inputs + def _get_layer_name(self, layer_name: LayerNameType | None) -> LayerNameType: + return self.encoded_layer_name if layer_name is None else layer_name + def pattern_non_fp8_quant_query( self, qkv: torch.Tensor, @@ -175,6 +266,7 @@ class QkNormRopeKvCachePattern: q_weight: torch.Tensor, k_weight: torch.Tensor, cos_sin_cache: torch.Tensor, + layer_name: LayerNameType | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size) @@ -190,7 +282,9 @@ class QkNormRopeKvCachePattern: q_rope = q_rope.view(-1, self.num_heads, self.head_size) k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size) v = v.view(-1, self.num_kv_heads, self.head_size_v) - dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name) + dummy = torch.ops.vllm.unified_kv_cache_update( + k_rope, v, self._get_layer_name(layer_name) + ) return dummy, q_rope, k_rope, v def replacement_non_fp8_quant_query( @@ -200,6 +294,7 @@ class QkNormRopeKvCachePattern: q_weight: torch.Tensor, k_weight: torch.Tensor, cos_sin_cache: torch.Tensor, + layer_name: LayerNameType | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: q_out = torch.empty( qkv.shape[0], @@ -228,10 +323,49 @@ class QkNormRopeKvCachePattern: rms_norm_eps=self.eps, cos_sin_cache=cos_sin_cache, is_neox=self.is_neox, - layer_name=self.layer_name, + layer_name=self._get_layer_name(layer_name), ) return results[0], results[1], results[2], v + def _quantize_query( + self, + query: torch.Tensor, + q_scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if self.query_quant_group_shape.is_per_tensor(): + q_out = torch.empty_like(query, dtype=current_platform.fp8_dtype()) + q_quant = auto_functionalized( + torch.ops.vllm.rocm_aiter_per_tensor_quant.default, + out=q_out, + x=query, + scale=q_scale, + is_dynamic=False, + ) + # The AITER op's schema marks scale mutable, so preserve its + # functionalized write-back as an output of the matched region. + return q_quant[1], q_quant[2] + + # QuantFP8.forward_hip falls back to the generic static op for the + # historical per-head shape (-1, block_size). This also covers the + # degenerate one-local-KV-head case where a scalar scale is represented + # as (-1, q_size), rather than changing Attention's classification. + q_out = torch.empty( + query.shape, + device=query.device, + dtype=current_platform.fp8_dtype(), + ) + q_quant = auto_functionalized( + torch.ops._C.static_scaled_fp8_quant.default, + result=q_out, + input=query, + scale=q_scale, + group_shape=[ + self.query_quant_group_shape.row, + self.query_quant_group_shape.col, + ], + ) + return q_quant[1], None + def pattern_fp8_quant_query( self, qkv: torch.Tensor, @@ -240,7 +374,8 @@ class QkNormRopeKvCachePattern: k_weight: torch.Tensor, cos_sin_cache: torch.Tensor, q_scale: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + layer_name: LayerNameType | None = None, + ) -> tuple[torch.Tensor, ...]: q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size) q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps) @@ -251,24 +386,15 @@ class QkNormRopeKvCachePattern: k_flat = k_normed.view(-1, self.k_size) q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache) - # Match the quant-query op Attention.forward inserts (fp8 KV + UNIFIED). - # Explicit auto_functionalized (out=[1]) keeps the quant node in the pattern. - q_out = torch.empty_like(q_rope, dtype=current_platform.fp8_dtype()) - q_quant = auto_functionalized( - torch.ops.vllm.rocm_aiter_per_tensor_quant.default, - out=q_out, - x=q_rope, - scale=q_scale, - is_dynamic=False, - ) - # `scale` is mutable: its copy_ write-back to _q_scale bumps the mutation - # region, so keep q flat (a reshape lands past the barrier and won't match). - q_rope_fp8 = q_quant[1] - q_scale_out = q_quant[2] + q_rope_fp8, q_scale_out = self._quantize_query(q_rope, q_scale) k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size) v = v.view(-1, self.num_kv_heads, self.head_size_v) - dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name) + dummy = torch.ops.vllm.unified_kv_cache_update( + k_rope, v, self._get_layer_name(layer_name) + ) + if q_scale_out is None: + return dummy, q_rope_fp8, k_rope, v return dummy, q_rope_fp8, k_rope, v, q_scale_out def replacement_fp8_quant_query( @@ -279,7 +405,8 @@ class QkNormRopeKvCachePattern: k_weight: torch.Tensor, cos_sin_cache: torch.Tensor, q_scale: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + layer_name: LayerNameType | None = None, + ) -> tuple[torch.Tensor, ...]: q_out = torch.empty( qkv.shape[0], self.num_heads, @@ -307,22 +434,14 @@ class QkNormRopeKvCachePattern: rms_norm_eps=self.eps, cos_sin_cache=cos_sin_cache, is_neox=self.is_neox, - layer_name=self.layer_name, + layer_name=self._get_layer_name(layer_name), ) - # Re-apply the quant on the kernel's bf16 q_out; fused op does not quant q. - # Same explicit auto_functionalized form as the pattern: [1] = quantized - # q, [2] = scale (returned so the buffer-writeback use is preserved). + # Re-apply the same quant form on the kernel's bf16 q_out; the fused + # operation does not quantize Q. q_fp8_flat = results[1].view(-1, self.q_size) - q_fp8_out = torch.empty_like(q_fp8_flat, dtype=current_platform.fp8_dtype()) - q_requant = auto_functionalized( - torch.ops.vllm.rocm_aiter_per_tensor_quant.default, - out=q_fp8_out, - x=q_fp8_flat, - scale=q_scale, - is_dynamic=False, - ) - q_fp8 = q_requant[1] # flat to mirror the pattern (see note above) - q_scale_out = q_requant[2] + q_fp8, q_scale_out = self._quantize_query(q_fp8_flat, q_scale) + if q_scale_out is None: + return results[0], q_fp8, results[2], v return results[0], q_fp8, results[2], v, q_scale_out @staticmethod @@ -345,6 +464,9 @@ class QkNormRopeKvCachePattern: view_to_reshape(gm) + def _extra_check(self, match: pm.Match) -> bool: + return True + def _register(self, pattern, replacement, pm_pass) -> None: trace_fn = QkNormRopeKvCachePattern.wrap_trace_fn( pm.fwd_only, @@ -370,40 +492,338 @@ class QkNormRopeKvCachePattern: inputs, trace_fn, pm_pass, + extra_check=self._extra_check, search_fn_pattern=search_fn_pattern, ) def register(self, pm_pass: PatternMatcherPass) -> None: # make_fx counts `self` in bound-method code params; wrap as plain fns. - # Distinct names per branch so mypy doesn't see one name, two signatures. if self.quant_query: + if _USE_LAYERNAME: - def pattern_q(qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale): - return self.pattern_fp8_quant_query( + def pattern_q_with_layer( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + q_scale, + layer_name, + ): + return self.pattern_fp8_quant_query( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + q_scale, + layer_name, + ) + + def replacement_q_with_layer( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + q_scale, + layer_name, + ): + return self.replacement_fp8_quant_query( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + q_scale, + layer_name, + ) + + self._register(pattern_q_with_layer, replacement_q_with_layer, pm_pass) + else: + + def pattern_q_without_layer( qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale - ) + ): + return self.pattern_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale + ) - def replacement_q( - qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale - ): - return self.replacement_fp8_quant_query( + def replacement_q_without_layer( qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale - ) + ): + return self.replacement_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale + ) - self._register(pattern_q, replacement_q, pm_pass) + self._register( + pattern_q_without_layer, replacement_q_without_layer, pm_pass + ) else: + if _USE_LAYERNAME: - def pattern_noq(qkv, positions, q_weight, k_weight, cos_sin_cache): - return self.pattern_non_fp8_quant_query( + def pattern_noq_with_layer( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + layer_name, + ): + return self.pattern_non_fp8_quant_query( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + layer_name, + ) + + def replacement_noq_with_layer( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + layer_name, + ): + return self.replacement_non_fp8_quant_query( + qkv, + positions, + q_weight, + k_weight, + cos_sin_cache, + layer_name, + ) + + self._register( + pattern_noq_with_layer, replacement_noq_with_layer, pm_pass + ) + else: + + def pattern_noq_without_layer( qkv, positions, q_weight, k_weight, cos_sin_cache + ): + return self.pattern_non_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache + ) + + def replacement_noq_without_layer( + qkv, positions, q_weight, k_weight, cos_sin_cache + ): + return self.replacement_non_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache + ) + + self._register( + pattern_noq_without_layer, + replacement_noq_without_layer, + pm_pass, ) - def replacement_noq(qkv, positions, q_weight, k_weight, cos_sin_cache): - return self.replacement_non_fp8_quant_query( - qkv, positions, q_weight, k_weight, cos_sin_cache - ) - self._register(pattern_noq, replacement_noq, pm_pass) +class QkNormMRopeKvCachePattern(QkNormRopeKvCachePattern): + FUSED_OP = torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default + + def __init__( + self, + layer: Attention, + eps: float, + is_neox: bool, + quant_query: bool, + rotary_dim: int, + mrope_section: tuple[int, int, int], + is_interleaved: bool, + ) -> None: + super().__init__( + layer=layer, + eps=eps, + is_neox=is_neox, + quant_query=quant_query, + ) + self.rotary_dim = rotary_dim + self.mrope_section = mrope_section + self.is_interleaved = is_interleaved + self.rope_matcher = MatcherMRotaryEmbedding( + is_neox=is_neox, + head_size=self.head_size, + rotary_dim=rotary_dim, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + mrope_section=mrope_section, + mrope_interleaved=is_interleaved, + ) + + def get_inputs(self) -> list[torch.Tensor]: + T = 5 + L = 4096 + qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size) + positions = empty_i64(3, T) + q_weight = empty_bf16(1, self.head_size) + k_weight = empty_bf16(1, self.head_size) + cos_sin_cache = empty_bf16(L, self.rotary_dim) + inputs = [qkv, positions, q_weight, k_weight, cos_sin_cache] + if self.quant_query: + inputs.append(empty_fp32(1)) + if _USE_LAYERNAME: + inputs.append(self.encoded_layer_name) + return inputs + + def _unfused_kv_views(self, qkv: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + # The supported backend reads K/V from the cache after the fused write. + # Keep view-compatible placeholders for the attention custom-op ABI. + _, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + k = k.view(qkv.shape[0], self.num_kv_heads, self.head_size) + v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v) + return k, v + + def _fused_results( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + layer_name: LayerNameType | None, + ) -> tuple[torch.Tensor, ...]: + q_out = torch.empty( + qkv.shape[0], + self.num_heads, + self.head_size, + device=qkv.device, + dtype=qkv.dtype, + ) + return auto_functionalized( + self.FUSED_OP, + q_out=q_out, + qkv=qkv, + positions=positions, + q_weight=q_weight, + k_weight=k_weight, + rms_norm_eps=self.eps, + cos_sin_cache=cos_sin_cache, + is_neox=self.is_neox, + mrope_section_t=self.mrope_section[0], + mrope_section_h=self.mrope_section[1], + mrope_section_w=self.mrope_section[2], + is_interleaved=self.is_interleaved, + rotary_dim=self.rotary_dim, + layer_name=self._get_layer_name(layer_name), + ) + + def replacement_non_fp8_quant_query( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + layer_name: LayerNameType | None = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + k, v = self._unfused_kv_views(qkv) + results = self._fused_results( + qkv, positions, q_weight, k_weight, cos_sin_cache, layer_name + ) + return results[0], results[1], k, v + + def replacement_fp8_quant_query( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + q_scale: torch.Tensor, + layer_name: LayerNameType | None = None, + ) -> tuple[torch.Tensor, ...]: + k, v = self._unfused_kv_views(qkv) + results = self._fused_results( + qkv, positions, q_weight, k_weight, cos_sin_cache, layer_name + ) + q_flat = results[1].view(-1, self.q_size) + q_fp8, q_scale_out = self._quantize_query(q_flat, q_scale) + if q_scale_out is None: + return results[0], q_fp8, k, v + return results[0], q_fp8, k, v, q_scale_out + + def _extra_check(self, match: pm.Match) -> bool: + expected = { + "head_size": self.head_size, + "rotary_dim": self.rotary_dim, + "mrope_section_t": self.mrope_section[0], + "mrope_section_h": self.mrope_section[1], + "mrope_section_w": self.mrope_section[2], + "mrope_interleaved": self.is_interleaved, + "is_neox_style": self.is_neox, + } + mrope_node = next( + (node for node in match.nodes if node.args and node.args[0] == MROPE_OP), + None, + ) + if mrope_node is None or any( + mrope_node.kwargs.get(name) != value for name, value in expected.items() + ): + return False + + # The fused AITER MRoPE entrypoint writes rotated K directly to the + # cache and does not expose a BF16 K result. Returning the raw K view is + # valid only for decoder unified attention, whose backend consumes K + # from that cache. Reject graphs that observe K in any other way. + output_nodes = match.output_nodes() + if len(output_nodes) < 3 or output_nodes[2] is None: + return False + matched_nodes = set(match.nodes) + attention_op = torch.ops.vllm.unified_attention_with_output.default + + def is_unified_attention_user(user: fx.Node) -> bool: + return user.op == "call_function" and ( + user.target == attention_op + or (user.args and user.args[0] == attention_op) + ) + + return all( + user in matched_nodes or is_unified_attention_user(user) + for user in output_nodes[2].users + ) + + +def _discover_mrope_configs( + config: VllmConfig, +) -> tuple[tuple[tuple[int, int, int], bool], ...]: + model_config = config.model_config + hf_configs = ( + getattr(model_config, "hf_text_config", None), + getattr(model_config, "hf_config", None), + ) + configs: set[tuple[tuple[int, int, int], bool]] = set() + for hf_config in hf_configs: + if hf_config is None: + continue + rope_parameters = getattr(hf_config, "rope_parameters", None) + if rope_parameters is None: + rope_parameters = getattr(hf_config, "rope_scaling", None) + if not isinstance(rope_parameters, Mapping): + continue + section = rope_parameters.get("mrope_section") + if not isinstance(section, (list, tuple)) or len(section) != 3: + continue + try: + parsed_section = ( + int(section[0]), + int(section[1]), + int(section[2]), + ) + except (TypeError, ValueError): + continue + if any(value <= 0 for value in parsed_section): + continue + configs.add( + ( + parsed_section, + bool(rope_parameters.get("mrope_interleaved", False)), + ) + ) + return tuple(sorted(configs)) # --------------------------------------------------------------------------- @@ -412,7 +832,7 @@ class QkNormRopeKvCachePattern: class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): - """Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel. + """Fuse QK-norm + RoPE/MRoPE + KV cache update into an AITER HIP kernel. Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for attention layers that support the combined operation, eliminating two @@ -429,22 +849,33 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): cc = config.compilation_config self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num + self.mrope_configs = ( + _discover_mrope_configs(config) + if cc.is_custom_op_enabled("rotary_embedding") + else () + ) dtype = config.model_config.dtype if dtype not in (torch.bfloat16, torch.float16): logger.warning_once( - "QK Norm+RoPE+KVCache fusion not enabled: unsupported dtype %s", dtype + "QK Norm+RoPE/MRoPE+KVCache fusion not enabled: unsupported dtype %s", + dtype, ) return attn_layers = get_layers_from_vllm_config(config, Attention) for _, layer in attn_layers.items(): - if not layer.impl.fused_qk_norm_rope_kvcache_supported(): + supports_rope = layer.impl.fused_qk_norm_rope_kvcache_supported() + supports_mrope = ( + bool(self.mrope_configs) + and layer.impl.fused_qk_norm_mrope_kvcache_supported() + ) + if not supports_rope and not supports_mrope: continue if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS: logger.warning_once( - "QK Norm+RoPE+KVCache fusion not enabled for a layer: " + "QK Norm+RoPE/MRoPE+KVCache fusion not enabled for a layer: " "head_size=%d is not supported by the " "fused_qk_norm_rope_cache_pts_quant_shuffle kernel " "(supported: %s). Falling back to the unfused path.", @@ -455,22 +886,71 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): if layer.head_size_v != layer.head_size: # The fused kernel uses a single head_dim for q/k/v. logger.warning_once( - "QK Norm+RoPE+KVCache fusion not enabled for a layer: " + "QK Norm+RoPE/MRoPE+KVCache fusion not enabled for a layer: " "head_size_v=%d differs from head_size=%d, which the fused " "kernel does not support. Falling back to the unfused path.", layer.head_size_v, layer.head_size, ) continue - for epsilon in [1e-5, 1e-6]: - for neox in [True, False]: - for quant_q in [False, True]: - QkNormRopeKvCachePattern( - layer=layer, - eps=epsilon, - is_neox=neox, - quant_query=quant_q, - ).register(self.patterns) + if supports_rope: + for epsilon in [1e-5, 1e-6]: + for neox in [True, False]: + for quant_q in [False, True]: + QkNormRopeKvCachePattern( + layer=layer, + eps=epsilon, + is_neox=neox, + quant_query=quant_q, + ).register(self.patterns) + + if supports_mrope: + for section, is_interleaved in self.mrope_configs: + rotary_dim = 2 * sum(section) + if rotary_dim > layer.head_size: + logger.warning_once( + "QK Norm+MRoPE+KVCache fusion not enabled for a " + "layer: rotary_dim=%d exceeds head_size=%d.", + rotary_dim, + layer.head_size, + ) + continue + vector_width = layer.head_size // 32 + for epsilon in [1e-5, 1e-6]: + for neox in [True, False]: + # Each warp lane processes `vector_width` adjacent + # values. NeoX additionally exchanges vectors + # across the half-rotary boundary. + rotary_alignment = vector_width * (2 if neox else 1) + if rotary_dim % rotary_alignment != 0: + logger.warning_once( + "QK Norm+MRoPE+KVCache fusion not enabled " + "for a layer/style: rotary_dim=%d must be " + "divisible by %d for head_size=%d and " + "is_neox=%s.", + rotary_dim, + rotary_alignment, + layer.head_size, + neox, + ) + continue + for quant_q in [False, True]: + QkNormMRopeKvCachePattern( + layer=layer, + eps=epsilon, + is_neox=neox, + quant_query=quant_q, + rotary_dim=rotary_dim, + mrope_section=section, + is_interleaved=is_interleaved, + ).register(self.patterns) + + # Opaque LayerName is a pattern wildcard on torch >= 2.11, so + # homogeneous attention layers produce duplicate search patterns. + # As in the other LayerName-aware fusion passes, one supported + # layer registers all shape/style variants for the model. + if _USE_LAYERNAME: + break self.dump_patterns(config, self.patterns) @@ -478,8 +958,8 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): def __call__(self, graph: fx.Graph) -> None: self.matched_count = self.patterns.apply(graph) logger.info( - "QK-Norm+RoPE+KVCache fusion: replaced %s pattern(s) " - "with AITER fused_qk_norm_rope_cache_pts_quant_shuffle", + "QK-Norm+RoPE/MRoPE+KVCache fusion: replaced %s pattern(s) " + "with an AITER fused attention-prologue kernel", self.matched_count, ) @@ -487,4 +967,10 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): return compile_range.end <= self.max_token_num def uuid(self) -> str: - return VllmInductorPass.hash_source(self, QkNormRopeKvCachePattern) + return VllmInductorPass.hash_source( + self, + QkNormRopeKvCachePattern, + QkNormMRopeKvCachePattern, + _discover_mrope_configs, + repr(self.mrope_configs), + ) diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 8ccc7332bd..eefcf1f49e 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -154,15 +154,15 @@ class PassConfig: fuse_rope_kvcache: bool = None # type: ignore[assignment] """Fuse the QK rope + KV cache ops.""" fuse_qk_norm_rope_kvcache: bool = Field(default=None) # type: ignore[assignment] - """Fuse QK RMSNorm + RoPE + KV cache update into a single AITER HIP + """Fuse QK RMSNorm + RoPE/MRoPE + KV cache update into an AITER HIP kernel. Supersedes both enable_qk_norm_rope_fusion and fuse_rope_kvcache - for layers that support it. Auto-enabled at O1+ on ROCm for models - with QK-norm (e.g. Qwen3-MoE).""" + for layers that support it. Auto-enabled at O2+ on ROCm for models + with QK-norm (e.g. Qwen3-MoE and Qwen3-VL-class architectures).""" rope_kvcache_fusion_max_token_num: int = 256 """The threshold for ROCm AITER RoPE+KVCache fusion e.g. for small batch decode. Larger batch sizes e.g. during prefill will use the unfused kernels. - Also applies to the fused QK-Norm+RoPE+KVCache pass. + Also applies to the fused QK-Norm+RoPE/MRoPE+KVCache pass. """ fi_allreduce_fusion_max_size_mb: float | None = None diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index e1c80be67b..54ff2d483b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -242,7 +242,7 @@ def enable_mla_dual_rms_norm_fusion(cfg: "VllmConfig") -> bool: def enable_qk_norm_rope_kvcache(cfg: "VllmConfig") -> bool: - """Enable fused QK-norm + RoPE + KV cache update on ROCm with AITER.""" + """Enable fused QK-norm + RoPE/MRoPE + KV cache update with AITER.""" from vllm._aiter_ops import rocm_aiter_ops if not rocm_aiter_ops.is_enabled(): @@ -776,6 +776,45 @@ class VllmConfig: architectures = set(model_config.architectures) return bool(architectures & default_breakable_cudagraph_architectures()) + def _uses_breakable_cudagraph_for_batch_invariance(self) -> bool: + """Avoid freezing runtime-M tile lookup in compiled forward (#54243). + Breakable graphs look up tuned bf16, unquantized qkv/o/gate_up/down tiles + at capture; lm_head runs outside compiled forward and does not benefit.""" + from vllm.model_executor.determinism import batch_invariant_configs as bi + from vllm.platforms import current_platform + + model = self.model_config + if ( + not envs.VLLM_BATCH_INVARIANT + or model is None + or model.enforce_eager + or model.dtype != torch.bfloat16 + or model.quantization is not None + or not current_platform.is_cuda() + # Sequence parallelism / async TP are torch.compile passes. + or self.compilation_config.pass_config.enable_sp + or self.compilation_config.pass_config.fuse_gemm_comms + ): + return False + family = bi._get_tuned_matmul_arch_family( + current_platform.get_device_capability() + ) + if family is None or family not in bi._BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: + return False + table = bi._BATCH_INVARIANT_MATMUL_TUNED_CONFIGS[family] + parallel = self.parallel_config + tp = parallel.tensor_parallel_size + hidden = model.get_hidden_size() + head = model.get_head_size() + heads = model.get_num_attention_heads(parallel) + kv_heads = model.get_num_kv_heads(parallel) + shapes = [((heads + 2 * kv_heads) * head, hidden), (hidden, heads * head)] + intermediate = getattr(model.hf_text_config, "intermediate_size", None) + # Per-layer sizes (e.g. Gemma3n) are not modeled. + if isinstance(intermediate, int): + shapes += [(2 * intermediate // tp, hidden), (hidden, intermediate // tp)] + return any(shape in table for shape in shapes) + def _maybe_enable_breakable_cudagraph(self) -> bool: if ( "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ @@ -786,6 +825,16 @@ class VllmConfig: "Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. " "Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out." ) + elif ( + "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ + and envs.VLLM_BATCH_INVARIANT + and self._uses_breakable_cudagraph_for_batch_invariance() + ): + os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1" + logger.info_once( + "VLLM_BATCH_INVARIANT=1: auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. " + "Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out." + ) from vllm.compilation.breakable_cudagraph import ( is_breakable_cudagraph_enabled, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index fbe85f46a3..ae9a85f0de 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -103,6 +103,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # (``_pop_done_transfers``); guarded by # ``_sending_transfers_lock``. self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list) + self._send_failures: set[ReqId] = set() self._sending_transfers_lock = threading.Lock() # Writer-thread owned matching state. @@ -162,6 +163,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): for handle in handles: self.nixl_wrapper.release_xfer_handle(handle) self._sending_transfers.clear() + self._send_failures.clear() super().shutdown() # --- Engine-main-thread entry point -------------------------------- # @@ -728,6 +730,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # don't have a ``_recving_metadata`` entry to invalidate, so # we just release the handle and let the engine reschedule # via the lease / watchdog. + with self._sending_transfers_lock: + self._send_failures.add(request_id) if not self._handle_failed_transfer(request_id, handle): return handle return None @@ -811,13 +815,16 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): done_pushing, failed_pushing = self._pop_done_transfers( self._sending_transfers ) - # A failed send must never be reported as done: its blocks - # are freed via the lease / watchdog instead. - done_pushing = { - req_id - for req_id in done_pushing - failed_pushing - if req_id in self._recving_metadata - } + # Remember failures until the final sibling WRITE completes. + self._send_failures.update(failed_pushing) + successful = { + req_id + for req_id in done_pushing - self._send_failures + if req_id in self._reqs_to_send or req_id in self._reqs_to_process + } + self._send_failures.difference_update(done_pushing | done_sending) + # Expired requests were already reported, even if their WRITEs finish later. + done_pushing = successful for req_id in done_pushing: self._reqs_to_send.pop(req_id, None) self._reqs_to_process.discard(req_id) diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 4d892d89d5..d7e235da1b 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -116,9 +116,7 @@ class OpenAIServingCompletion(GenerateBaseServing): See https://platform.openai.com/docs/api-reference/completions/create for the API specification. This API mimics the OpenAI Completion API. - NOTE: Currently we do not support the following feature: - - suffix (the language models we currently support do not support - suffix) + NOTE: suffix is only supported by models that implement FIM rendering. """ return await self._with_kv_transfer_rejection_cleanup( self._create_completion(request, raw_request), request, raw_request diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index b98dfee18b..4655181f78 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -31,11 +31,14 @@ class HummingLinearKernel(MPLinearKernel): ) name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name} + if self.w_zp_name is not None and hasattr(layer, self.w_zp_name): + name_map["zero_point"] = self.w_zp_name group_size = self.config.group_size quant_config = { "quant_method": "humming", "dtype": "int" + str(self.config.weight_type.size_bits), "group_size": 0 if group_size == -1 else group_size, + "has_zero_point": self.config.zero_points, } if self.config.zero_points: diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 1e21a4fe2c..ee3b9b3f4b 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -337,6 +337,25 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): ) intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K)) + # Under EP, drop the top-k slots routed to remote experts at alignment + # time instead of launching `off_experts == -1` GEMM blocks that only + # write zeros; the pad-aware `moe_sum` below skips the same slots. + # Their rows in the workspace caches are then never written, so every + # consumer of those caches must be row-local: a dynamic per-tensor + # scale for the second GEMM's activation would take its amax over the + # untouched rows, and LoRA aligns the full `topk_ids` itself. + a2_scale_is_global = ( + self.quant_dtype is not None + and a2_scale is None + and self.block_shape is None + and (not self.per_act_token_quant or self.quantization_emulation) + ) + skip_invalid = ( + expert_map is not None + and self._lora_context is None + and not a2_scale_is_global + ) + # Include fused shared-expert rows while preserving EP remapping. num_align_experts = w1.shape[0] if expert_map is None else global_num_experts sorted_token_ids, expert_ids, num_tokens_post_padded = ( @@ -350,6 +369,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): use_int8_w8a16=self.quant_config.use_int8_w8a16, use_int4_w4a16=self.quant_config.use_int4_w4a16, block_shape=self.block_shape, + ignore_invalid_experts=skip_invalid, ) ) @@ -585,10 +605,21 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): ) # separate function is required for MoE + LoRA - self.moe_sum(intermediate_cache3, output) + self.moe_sum(intermediate_cache3, output, topk_ids, expert_map) - def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None: - ops.moe_sum(input, output) + def moe_sum( + self, + input: torch.Tensor, + output: torch.Tensor, + topk_ids: torch.Tensor | None = None, + expert_map: torch.Tensor | None = None, + ) -> None: + if expert_map is not None: + # Skip the slots whose expert is not on this rank: the rows the + # alignment dropped, or the zeros the `-1` blocks wrote. + ops.moe_sum(input, output, topk_ids, expert_map) + else: + ops.moe_sum(input, output) class TritonWNA16Experts(TritonExperts): diff --git a/vllm/model_executor/layers/fused_moe/router/dsv4_topk.py b/vllm/model_executor/layers/fused_moe/router/dsv4_topk.py index dbd29f3d08..1df95bc611 100644 --- a/vllm/model_executor/layers/fused_moe/router/dsv4_topk.py +++ b/vllm/model_executor/layers/fused_moe/router/dsv4_topk.py @@ -41,6 +41,7 @@ if current_platform.is_cuda(): def _dsv4_topk_kernel( gating_output_ptr, correction_bias_ptr, + is_padding_ptr, topk_weights_ptr, topk_ids_ptr, routed_scaling_factor, @@ -49,6 +50,7 @@ if current_platform.is_cuda(): image_sentinel_lo, NUM_EXPERTS: tl.constexpr, BLOCK_N: tl.constexpr, + HAS_PADDING: tl.constexpr, HAS_VL: tl.constexpr, launch_pdl: tl.constexpr, ): @@ -106,6 +108,11 @@ if current_platform.is_cuda(): output_mask = topk_offsets < 6 output_offsets = row * 6 + topk_offsets + if HAS_PADDING: + is_padding = tl.load(is_padding_ptr + row) + selected_weights = tl.where(is_padding, 0.0, selected_weights) + selected_ids = tl.where(is_padding, -1, selected_ids) + if launch_pdl: tl.extra.cuda.gdc_launch_dependents() @@ -118,11 +125,20 @@ def dsv4_topk( correction_bias: torch.Tensor, indices_dtype: torch.dtype, routed_scaling_factor: float, + is_padding: torch.Tensor | None = None, input_ids: torch.Tensor | None = None, bias_vl: torch.Tensor | None = None, image_sentinel_lo: int = 0, ) -> tuple[torch.Tensor, torch.Tensor]: num_tokens, num_experts = gating_output.shape + if is_padding is not None: + assert is_padding.dtype == torch.bool + assert is_padding.shape == (num_tokens,) + assert is_padding.device == gating_output.device + assert is_padding.is_contiguous() + assert indices_dtype in (torch.int32, torch.int64), ( + "Padding requires a signed indices dtype for the -1 sentinel." + ) has_vl = bias_vl is not None and image_sentinel_lo > 0 if bias_vl is not None: assert input_ids is not None, "bias_vl routing requires input_ids" @@ -136,6 +152,7 @@ def dsv4_topk( _dsv4_topk_kernel[(num_tokens,)]( gating_output, correction_bias, + is_padding, topk_weights, topk_ids, routed_scaling_factor, @@ -144,6 +161,7 @@ def dsv4_topk( image_sentinel_lo, NUM_EXPERTS=num_experts, BLOCK_N=triton.next_power_of_2(num_experts), + HAS_PADDING=is_padding is not None, HAS_VL=has_vl, num_warps=1, launch_pdl=current_platform.is_arch_support_pdl(), diff --git a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py index 257ede27d5..d464d73ba1 100644 --- a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py +++ b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py @@ -148,9 +148,13 @@ def fused_topk_bias( ) output_indices_dtype = torch.int32 if indices_type is None else indices_type + padding_mask = _get_padding_mask(gating_output.shape[0]) if ( scoring_func == "sqrtsoftplus" and hash_indices_table is None + # dsv4_topk marks padded rows with a -1 sentinel, which unsigned + # indices cannot represent; keep the generic path for that pair. + and (padding_mask is None or output_indices_dtype != torch.uint32) and can_use_dsv4_topk( gating_output, e_score_correction_bias, @@ -165,6 +169,7 @@ def fused_topk_bias( e_score_correction_bias, output_indices_dtype, routed_scaling_factor, + is_padding=padding_mask, input_ids=input_tokens, bias_vl=bias_vl, image_sentinel_lo=image_sentinel_lo, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 3603fd8bde..a4cd5d4d13 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -787,6 +787,7 @@ class CompressedTensorsConfig(QuantizationConfig): num_bits=weight_quant.num_bits, strategy=weight_quant.strategy, group_size=weight_quant.group_size, + symmetric=weight_quant.symmetric, input_quant=input_quant, layer_name=layer_name, quant_format=format, @@ -799,6 +800,7 @@ class CompressedTensorsConfig(QuantizationConfig): num_bits=weight_quant.num_bits, strategy=weight_quant.strategy, group_size=weight_quant.group_size, + symmetric=weight_quant.symmetric, input_quant=input_quant, layer_name=layer_name, quant_format=format, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py index 42688ab834..8cfce07544 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py @@ -42,7 +42,15 @@ WNA16_SUPPORTED_TYPES_MAP = { 7: scalar_types.uint7b64, 8: scalar_types.uint8b128, } -WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8} +WNA16_ZP_SUPPORTED_TYPES_MAP = { + 2: scalar_types.uint2, + 3: scalar_types.uint3, + 4: scalar_types.uint4, + 5: scalar_types.uint5, + 6: scalar_types.uint6, + 7: scalar_types.uint7, + 8: scalar_types.uint8, +} WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys()) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py index 427c92310f..acd4a96701 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming. +"""Weight N-bit INT scheme with INT4 activation quant via Humming. Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit) with INT4 symmetric dynamic per-token/per-group input activation -quantization. Static, per-tensor, and asymmetric activation quantization -are not supported. +quantization. Supports both symmetric and asymmetric weight quantization. +Static, per-tensor, and asymmetric activation quantization are not supported. """ import math @@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( ) from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501 WNA16_SUPPORTED_TYPES_MAP, + WNA16_ZP_SUPPORTED_TYPES_MAP, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( marlin_repeat_scales_on_all_ranks, @@ -36,6 +37,7 @@ from vllm.model_executor.parameter import ( BasevLLMParameter, ChannelQuantScaleParameter, GroupQuantScaleParameter, + PackedColumnParameter, PackedvLLMParameter, ) @@ -52,6 +54,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): num_bits: int, strategy: str, group_size: int | None = None, + symmetric: bool = True, input_quant: QuantizationArgs | None = None, layer_name: str | None = None, quant_format: str = "pack-quantized", @@ -60,6 +63,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): self.pack_factor = Fraction(32, num_bits) self.strategy = strategy self.group_size = -1 if group_size is None else group_size + self.symmetric = symmetric self.input_quant = input_quant self.layer_name = layer_name self.quant_format = quant_format @@ -69,7 +73,19 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): f"Unsupported num_bits = {num_bits} for WNA4Int; " f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}" ) - self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Asymmetric quantization not supported for " + f"num_bits = {num_bits}. Supported: " + f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}" + ) + + self.quant_type = ( + WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits] + if not self.symmetric + else WNA16_SUPPORTED_TYPES_MAP[num_bits] + ) if input_quant is not None: if not input_quant.symmetric: @@ -143,7 +159,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): weight_type=self.quant_type, act_type=params_dtype, group_size=self.group_size, - zero_points=False, + zero_points=not self.symmetric, ) kernel_type = choose_mp_linear_kernel(mp_config) @@ -151,12 +167,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__) self._kernel_backends_being_used.add(kernel_type.__name__) - self.kernel = kernel_type( - mp_config, - w_q_param_name="weight_packed", - w_s_param_name="weight_scale", - ) - input_quant_config = self._build_input_quant_config() if input_quant_config is not None: layer._humming_input_quant_config = input_quant_config @@ -167,10 +177,10 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): partition_scales = not marlin_repeat_scales_on_all_ranks( self.group_size, row_parallel ) - scales_size = input_size // group_size + scales_and_zp_size = input_size // group_size if partition_scales: assert input_size_per_partition % group_size == 0 - scales_size = input_size_per_partition // group_size + scales_and_zp_size = input_size_per_partition // group_size packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) layer.register_parameter( @@ -190,7 +200,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ) scale_data = torch.empty( - output_size_per_partition, scales_size, dtype=params_dtype + output_size_per_partition, scales_and_zp_size, dtype=params_dtype ) if partition_scales: weight_scale = GroupQuantScaleParameter( @@ -207,6 +217,34 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ) layer.register_parameter("weight_scale", weight_scale) + if not self.symmetric: + packed_output_dim = math.ceil( + output_size_per_partition * self.num_bits / 32 + ) + zeros_data = torch.zeros( + packed_output_dim, + scales_and_zp_size, + dtype=torch.int32, + ) + if not partition_scales: + qzeros = PackedColumnParameter( + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + else: + qzeros = PackedvLLMParameter( + input_dim=1, + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + layer.register_parameter("weight_zero_point", qzeros) + layer.register_parameter( "weight_shape", BasevLLMParameter( @@ -215,6 +253,13 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme): ), ) + self.kernel = kernel_type( + mp_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point", + ) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py index a14077d899..156f29908d 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming. +"""Weight N-bit INT scheme with INT8 activation quant via Humming. Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit) with INT8 symmetric dynamic per-token/per-group input activation -quantization. Static, per-tensor, and asymmetric activation quantization -are not supported. +quantization. Supports both symmetric and asymmetric weight quantization. +Static, per-tensor, and asymmetric activation quantization are not supported. """ import math @@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( ) from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501 WNA16_SUPPORTED_TYPES_MAP, + WNA16_ZP_SUPPORTED_TYPES_MAP, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( marlin_repeat_scales_on_all_ranks, @@ -36,6 +37,7 @@ from vllm.model_executor.parameter import ( BasevLLMParameter, ChannelQuantScaleParameter, GroupQuantScaleParameter, + PackedColumnParameter, PackedvLLMParameter, ) @@ -52,6 +54,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): num_bits: int, strategy: str, group_size: int | None = None, + symmetric: bool = True, input_quant: QuantizationArgs | None = None, layer_name: str | None = None, quant_format: str = "pack-quantized", @@ -60,6 +63,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): self.pack_factor = Fraction(32, num_bits) self.strategy = strategy self.group_size = -1 if group_size is None else group_size + self.symmetric = symmetric self.input_quant = input_quant self.layer_name = layer_name self.quant_format = quant_format @@ -69,7 +73,19 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): f"Unsupported num_bits = {num_bits} for WNA8Int; " f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}" ) - self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits] + + if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP: + raise ValueError( + f"Asymmetric quantization not supported for " + f"num_bits = {num_bits}. Supported: " + f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}" + ) + + self.quant_type = ( + WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits] + if not self.symmetric + else WNA16_SUPPORTED_TYPES_MAP[num_bits] + ) if input_quant is not None: if not input_quant.symmetric: @@ -143,7 +159,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): weight_type=self.quant_type, act_type=params_dtype, group_size=self.group_size, - zero_points=False, + zero_points=not self.symmetric, ) kernel_type = choose_mp_linear_kernel(mp_config) @@ -151,12 +167,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__) self._kernel_backends_being_used.add(kernel_type.__name__) - self.kernel = kernel_type( - mp_config, - w_q_param_name="weight_packed", - w_s_param_name="weight_scale", - ) - input_quant_config = self._build_input_quant_config() if input_quant_config is not None: layer._humming_input_quant_config = input_quant_config @@ -167,10 +177,10 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): partition_scales = not marlin_repeat_scales_on_all_ranks( self.group_size, row_parallel ) - scales_size = input_size // group_size + scales_and_zp_size = input_size // group_size if partition_scales: assert input_size_per_partition % group_size == 0 - scales_size = input_size_per_partition // group_size + scales_and_zp_size = input_size_per_partition // group_size packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32) layer.register_parameter( @@ -190,7 +200,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ) scale_data = torch.empty( - output_size_per_partition, scales_size, dtype=params_dtype + output_size_per_partition, scales_and_zp_size, dtype=params_dtype ) if partition_scales: weight_scale = GroupQuantScaleParameter( @@ -207,6 +217,34 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ) layer.register_parameter("weight_scale", weight_scale) + if not self.symmetric: + packed_output_dim = math.ceil( + output_size_per_partition * self.num_bits / 32 + ) + zeros_data = torch.zeros( + packed_output_dim, + scales_and_zp_size, + dtype=torch.int32, + ) + if not partition_scales: + qzeros = PackedColumnParameter( + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + else: + qzeros = PackedvLLMParameter( + input_dim=1, + output_dim=0, + packed_dim=0, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + data=zeros_data, + ) + layer.register_parameter("weight_zero_point", qzeros) + layer.register_parameter( "weight_shape", BasevLLMParameter( @@ -215,6 +253,13 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme): ), ) + self.kernel = kernel_type( + mp_config, + w_q_param_name="weight_packed", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point", + ) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index a3a3a36f00..2be85d8176 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -5,7 +5,6 @@ import fnmatch from typing import TYPE_CHECKING, Any, NamedTuple, cast import torch -from transformers import PretrainedConfig from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention @@ -82,10 +81,6 @@ __all__ = ["QuarkLinearMethod"] logger = init_logger(__name__) -# model_type values that use dynamic MXFP4 re-quantization for -# OCP MX fp4 Quark checkpoints -_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"}) - class QuantKeyMatch(NamedTuple): matches: bool @@ -119,37 +114,6 @@ class QuarkConfig(QuantizationConfig): self.kv_cache_group = kv_cache_group self.kv_cache_config = kv_cache_config self.pack_method = pack_method - # Note : this flag is kept disabled because the overhead of - # dynamic mxfp4 quantization negates the performance gains - # that come from shifting to mxfp4. It is left here in case - # we want to re-enable it in the future. - self.dynamic_mxfp4_quant = False - - def maybe_update_config( - self, - model_name: str, - hf_config: PretrainedConfig | None = None, - revision: str | None = None, - ): - """Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints.""" - if hf_config is None: - return - - if ( - getattr(hf_config, "model_type", None) - not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES - ): - return - - quant_config = getattr(hf_config, "quantization_config", None) - if isinstance(quant_config, dict): - quant_dtype = ( - quant_config.get("global_quant_config", {}) - .get("weight", {}) - .get("dtype") - ) - if quant_dtype == "fp4": - self.dynamic_mxfp4_quant = True def get_linear_method(self) -> "QuarkLinearMethod": return QuarkLinearMethod(self) @@ -201,17 +165,6 @@ class QuarkConfig(QuantizationConfig): self.get_quant_method_target(prefix, type(layer)) ) - exclude_layers = cast(list[str], self.quant_config.get("exclude")) - is_ignored = should_ignore_layer( - prefix, - ignore=exclude_layers, - fused_mapping=self.packed_modules_mapping, - check_children=isinstance(layer, RoutedExperts), - ) - dynamic_mxfp4_quant = ( - is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant - ) - if method_cls is UnquantizedFusedMoEMethod: return UnquantizedFusedMoEMethod(layer.moe_config) if method_cls is UnquantizedLinearMethod: @@ -222,7 +175,6 @@ class QuarkConfig(QuantizationConfig): scheme_cls, weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, - dynamic_mxfp4_quant=dynamic_mxfp4_quant, weight_config=self._find_matched_config(prefix, type(layer)).get( "weight" ), @@ -274,11 +226,6 @@ class QuarkConfig(QuantizationConfig): if is_routed_experts: return None, None, UnquantizedFusedMoEMethod if issubclass(layer_type, LinearBase): - if "self_attn" in prefix and self.dynamic_mxfp4_quant: - weight_quant_key, activation_key, _ = self.get_scheme_cls( - layer_type, prefix - ) - return weight_quant_key, activation_key, QuarkLinearMethod return None, None, UnquantizedLinearMethod return None, None, None if issubclass(layer_type, LinearBase): @@ -956,7 +903,6 @@ class QuarkConfig(QuantizationConfig): scheme_cls: type["QuarkScheme"], weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, - dynamic_mxfp4_quant: bool = False, weight_config: dict[str, Any] | None = None, ) -> "QuarkScheme": """Construct a Quark scheme selected by get_scheme_cls.""" @@ -996,9 +942,6 @@ class QuarkConfig(QuantizationConfig): ): kwargs["weight_quant_key"] = weight_quant_key - if scheme_cls is QuarkOCP_MX: - kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant - scheme = scheme_cls(**kwargs) # Raise error if device does not support the scheme diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py index 30758782c3..50263863d8 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py @@ -26,10 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.model_executor.parameter import ( GroupQuantScaleParameter, - ModelWeightParameter, PackedvLLMParameter, ) -from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform from .quark_scheme import QuarkScheme @@ -46,10 +44,9 @@ class QuarkOCP_MX(QuarkScheme): self, weight_quant_key: QuantKey, activation_quant_key: QuantKey | None, - dynamic_mxfp4_quant: bool = False, ): super().__init__(weight_quant_key, activation_quant_key) - self.dynamic_mxfp4_quant = dynamic_mxfp4_quant + self.weight_dtype = next( dtype for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items() @@ -109,21 +106,9 @@ class QuarkOCP_MX(QuarkScheme): def get_min_capability(cls) -> int: return 70 - def process_dynamic_mxfp4_weights_after_loading( - self, layer: torch.nn.Module - ) -> None: - from aiter.ops.triton.quant import dynamic_mxfp4_quant - - w_q, w_s = dynamic_mxfp4_quant(layer.weight) - layer.weight_scale = torch.nn.Parameter(w_s, requires_grad=False) - layer.weight = torch.nn.Parameter(w_q, requires_grad=False) - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False) - if self.dynamic_mxfp4_quant: - self.process_dynamic_mxfp4_weights_after_loading(layer) - self.ocp_mx_linear.process_weights_after_loading(layer) def create_weights( @@ -145,51 +130,36 @@ class QuarkOCP_MX(QuarkScheme): "tensor-parallel sharding for this layer." ) - if self.dynamic_mxfp4_quant: - weight = ModelWeightParameter( - data=torch.empty( - sum(output_partition_sizes), - input_size_per_partition, - dtype=params_dtype, - ), - input_dim=1, - output_dim=0, - weight_loader=weight_loader, - ) + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes - layer.register_parameter("weight", weight) - set_weight_attrs(weight, kwargs) - else: - output_size_per_partition = sum(output_partition_sizes) - layer.logical_widths = output_partition_sizes + # WEIGHT + weight = PackedvLLMParameter( + data=torch.empty( + output_size_per_partition, + self.get_packed_dim(input_size_per_partition, self.weight_dtype), + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + packed_dim=1, + packed_factor=self.packed_factor, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) - # WEIGHT - weight = PackedvLLMParameter( - data=torch.empty( - output_size_per_partition, - self.get_packed_dim(input_size_per_partition, self.weight_dtype), - dtype=torch.uint8, - ), - input_dim=1, - output_dim=0, - packed_dim=1, - packed_factor=self.packed_factor, - weight_loader=weight_loader, - ) - layer.register_parameter("weight", weight) - - # WEIGHT SCALE - weight_scale = GroupQuantScaleParameter( - data=torch.empty( - output_size_per_partition, - input_size_per_partition // OCP_MX_BLOCK_SIZE, - dtype=torch.uint8, - ), - input_dim=1, - output_dim=0, - weight_loader=weight_loader, - ) - layer.register_parameter("weight_scale", weight_scale) + # WEIGHT SCALE + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // OCP_MX_BLOCK_SIZE, + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale", weight_scale) if self.weight_quant_key == kMxfp4Static: self.ocp_mx_linear = init_mxfp4_linear_kernel( diff --git a/vllm/model_executor/layers/quantization/utils/humming/linear.py b/vllm/model_executor/layers/quantization/utils/humming/linear.py index 339ff640b0..0c64645c43 100644 --- a/vllm/model_executor/layers/quantization/utils/humming/linear.py +++ b/vllm/model_executor/layers/quantization/utils/humming/linear.py @@ -101,6 +101,19 @@ def prepare_humming_linear_layer_config( param_dtype=layer.params_dtype, ) + if ( + weight_schema.has_zero_point + and input_schema.a_dtype is not None + and weight_schema.b_dtype.num_bits == input_schema.a_dtype.num_bits + ): + raise ValueError( + "Humming does not support asymmetric (zero_point) weights " + "when weight and activation bit widths are equal " + f"(W{weight_schema.b_dtype.num_bits}" + f"A{input_schema.a_dtype.num_bits}). " + "Use symmetric weights or a different activation bit width." + ) + tensors.update(input_tensors) layer.weight_schema = weight_schema input_schema = check_and_fallback_input_schema( diff --git a/vllm/model_executor/layers/rotary_embedding/mrope.py b/vllm/model_executor/layers/rotary_embedding/mrope.py index 185bfd650c..42013926d7 100644 --- a/vllm/model_executor/layers/rotary_embedding/mrope.py +++ b/vllm/model_executor/layers/rotary_embedding/mrope.py @@ -7,6 +7,7 @@ import torch from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op from .base import RotaryEmbeddingBase from .yarn_scaling_rope import YaRNScalingRotaryEmbedding, yarn_get_mscale @@ -240,6 +241,62 @@ def apply_interleaved_rope(x: torch.Tensor, mrope_section: list[int]) -> torch.T return torch.where(is_width, x[2], result) +def _mrope_apply( + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + cos_sin_cache: torch.Tensor, + head_size: int, + rotary_dim: int, + mrope_section_t: int, + mrope_section_h: int, + mrope_section_w: int, + mrope_interleaved: bool, + is_neox_style: bool, +) -> None: + cos_sin = cos_sin_cache[positions] + cos, sin = cos_sin.chunk(2, dim=-1) + query_out, key_out = triton_mrope( + query, + key, + cos, + sin, + [mrope_section_t, mrope_section_h, mrope_section_w], + head_size, + rotary_dim, + mrope_interleaved, + is_neox_style, + ) + if query_out.data_ptr() != query.data_ptr(): + query.copy_(query_out.view_as(query)) + if key_out.data_ptr() != key.data_ptr(): + key.copy_(key_out.view_as(key)) + + +def _mrope_apply_fake( + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + cos_sin_cache: torch.Tensor, + head_size: int, + rotary_dim: int, + mrope_section_t: int, + mrope_section_h: int, + mrope_section_w: int, + mrope_interleaved: bool, + is_neox_style: bool, +) -> None: + return + + +direct_register_custom_op( + op_name="mrope", + op_func=_mrope_apply, + mutates_args=["query", "key"], + fake_impl=_mrope_apply_fake, +) + + class MRotaryEmbedding(RotaryEmbeddingBase): """Rotary Embedding with Multimodal Sections.""" @@ -426,6 +483,35 @@ class MRotaryEmbedding(RotaryEmbeddingBase): key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) return query, key + def forward_hip( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor | None = None, + offsets: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if positions.ndim == 2: + assert key is not None + assert self.mrope_section + query_shape = query.shape + key_shape = key.shape + cos_sin_cache = self._match_cos_sin_cache_dtype(query) + torch.ops.vllm.mrope( + positions, + query, + key, + cos_sin_cache, + self.head_size, + self.rotary_dim, + self.mrope_section[0], + self.mrope_section[1], + self.mrope_section[2], + self.mrope_interleaved, + self.is_neox_style, + ) + return query.reshape(query_shape), key.reshape(key_shape) + return super().forward_hip(positions, query, key) + def forward_cpu( self, positions: torch.Tensor, diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 390c0dbfed..359195d4c2 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -498,11 +498,11 @@ class CpuPlatform(Platform): return # reconcile attention and mamba page sizes - backend_cls = cls._find_non_ssm_backend(vllm_config) - if backend_cls is None: + backend_classes = cls._find_non_ssm_backends(vllm_config) + if not backend_classes: return - cls._align_hybrid_block_size(vllm_config, backend_cls) + cls._align_hybrid_block_size(vllm_config, backend_classes[0]) @classmethod def discover_numa_topology(cls) -> list[list[int]]: diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 13e5b55402..76d96846e1 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -3,6 +3,8 @@ import contextlib import enum import functools +import itertools +import math import os import platform import sys @@ -573,10 +575,10 @@ class Platform: pass @classmethod - def _find_non_ssm_backend( + def _find_non_ssm_backends( cls, vllm_config: "VllmConfig" - ) -> "type[AttentionBackend] | None": - """Find the first non-SSM attention backend from model layers.""" + ) -> "list[type[AttentionBackend]]": + """Distinct non-SSM attention backends, in layer order.""" from vllm.config.vllm import get_layers_from_vllm_config from vllm.model_executor.layers.attention_layer_base import ( AttentionLayerBase, @@ -586,11 +588,62 @@ class Platform: vllm_config, AttentionLayerBase, # type: ignore[type-abstract] ) + backends: list[type[AttentionBackend]] = [] for layer in attn_layers.values(): b = layer.get_attn_backend() - if not b.is_ssm(): - return b - return None + if not b.is_ssm() and b not in backends: + backends.append(b) + return backends + + @classmethod + def _preferred_block_size_for_backends( + cls, + backend_classes: "list[type[AttentionBackend]]", + default_block_size: int, + vllm_config: "VllmConfig", + ) -> int: + """Smallest block size every backend accepts. + + ``supports_block_size`` may be overridden to accept exact sizes only + (CPU_MLA takes 16 and no multiple of it), so candidates are the LCMs of + one supported size per backend, each checked against every backend. + + Raises: + ValueError: If the backends share no supported block size. + + """ + from vllm.config.vllm import set_current_vllm_config + from vllm.v1.attention.backend import MultipleOf + + # Backends may read the current config to decide a preference. + with set_current_vllm_config(vllm_config): + if len(backend_classes) == 1: + return backend_classes[0].get_preferred_block_size(default_block_size) + if all(b.supports_block_size(default_block_size) for b in backend_classes): + return default_block_size + # A backend declaring no sizes accepts any, so it contributes 1. + per_backend_sizes = [ + [ + s.base if isinstance(s, MultipleOf) else s + for s in b.get_supported_kernel_block_sizes() + ] + or [1] + for b in backend_classes + ] + candidates = sorted( + {math.lcm(*sizes) for sizes in itertools.product(*per_backend_sizes)} + ) + for candidate in candidates: + if all(b.supports_block_size(candidate) for b in backend_classes): + return candidate + raise ValueError( + "The attention backends share no supported KV cache block size (" + + "; ".join( + f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" + for b in backend_classes + ) + + ")." + ) @classmethod def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: @@ -598,7 +651,6 @@ class Platform: For hybrid models, also aligns block_size with mamba page sizes. """ from vllm.config.cache import CacheConfig - from vllm.config.vllm import set_current_vllm_config cache_config = vllm_config.cache_config model_config = vllm_config.model_config @@ -607,34 +659,36 @@ class Platform: if not model_config: return - backend_cls = cls._find_non_ssm_backend(vllm_config) - if backend_cls is None: + backend_classes = cls._find_non_ssm_backends(vllm_config) + if not backend_classes: return - # Phase 1: Pick block size from backend (skip if user set --block-size) + # Phase 1: Pick a block size every attention backend supports (skip if + # user set --block-size). Models can mix backends with disjoint + # preferences, and a size from the first alone later fails + # select_common_block_size(). if not cache_config.user_specified_block_size: - with set_current_vllm_config(vllm_config): - preferred = backend_cls.get_preferred_block_size( - CacheConfig.DEFAULT_BLOCK_SIZE - ) + preferred = cls._preferred_block_size_for_backends( + backend_classes, CacheConfig.DEFAULT_BLOCK_SIZE, vllm_config + ) if preferred != CacheConfig.DEFAULT_BLOCK_SIZE: logger.info( - "Setting kv cache block size to %d for %s backend.", + "Setting kv cache block size to %d for %s backend(s).", preferred, - backend_cls.get_name(), + "/".join(b.get_name() for b in backend_classes), ) cache_config.block_size = preferred # Phase 2: Align block/mamba sizes for hybrid models # (may override user settings). if model_config.is_hybrid: - cls._align_hybrid_block_size(vllm_config, backend_cls) + cls._align_hybrid_block_size(vllm_config, backend_classes[0]) # Phase 3: Align block/page sizes when multiple KV dtypes share the # block pool (e.g. nvfp4 primary + unquantized skip layers). # May override the user's --block-size. if cache_config.kv_cache_dtype_skip_layers: - cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls) + cls._align_heterogeneous_kv_block_size(vllm_config, backend_classes[0]) @classmethod def _align_heterogeneous_kv_block_size( diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 9089d025e0..844fe4884b 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib.metadata import os import platform from datetime import timedelta @@ -361,6 +362,25 @@ def get_cdna_version() -> int: return 0 +@cache +def get_rocm_version() -> tuple[int, ...] | None: + """Return the installed ROCm release as (major, minor, patch), or None.""" + # ROCm 10+ ships as the `rocm` pip SDK; older releases install to /opt/rocm. + try: + version = importlib.metadata.version("rocm") + except importlib.metadata.PackageNotFoundError: + rocm_path = os.environ.get("ROCM_PATH", "/opt/rocm") + try: + with open(os.path.join(rocm_path, ".info", "version")) as f: + version = f.read() + except OSError: + return None + match = re.match(r"\s*(\d+)\.(\d+)(?:\.(\d+))?", version) + if match is None: + return None + return tuple(int(part) for part in match.groups() if part is not None) + + # Enable HIP online tuning early, before hipBLASLt initializes. # Turn on hipBLASLt online tuning if use AITER hipBLASLt GEMM. if ( diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 53e3f50b4e..87d11b7b92 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]): return tokenizer + def render_completion_suffix(self, prompt: str, suffix: str) -> str | None: + """Render OpenAI completion suffix input when the renderer supports FIM.""" + return None + def _decode(self, *args, **kwargs): return self.get_tokenizer().decode(*args, **kwargs) diff --git a/vllm/renderers/deepseek_v4.py b/vllm/renderers/deepseek_v4.py index ede2e9530c..470e21f1eb 100644 --- a/vllm/renderers/deepseek_v4.py +++ b/vllm/renderers/deepseek_v4.py @@ -16,6 +16,10 @@ from .inputs import DictPrompt from .inputs.preprocess import parse_dec_only_prompt from .params import ChatParams +_FIM_BEGIN = "<|fim▁begin|>" +_FIM_HOLE = "<|fim▁hole|>" +_FIM_END = "<|fim▁end|>" + class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): def __init__( @@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): def _apply_chat_template(self, *args, **kwargs): return self.get_tokenizer().apply_chat_template(*args, **kwargs) + def render_completion_suffix(self, prompt: str, suffix: str) -> str | None: + return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}" + def render_messages( self, messages: list[ChatCompletionMessageParam], diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index f4d21dedad..e5169d70ba 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -4,7 +4,7 @@ import time from collections.abc import Sequence from dataclasses import dataclass from http import HTTPStatus -from typing import TYPE_CHECKING, Any, TypeAlias +from typing import TYPE_CHECKING, Any, TypeAlias, cast from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem from openai.types.responses.tool import Mcp, Tool @@ -548,9 +548,58 @@ class OnlineRenderer: Called directly by render_completion_request and delegated to by OpenAIServingCompletion.render_completion_request after its engine-aware checks. """ - # Return error for unsupported features. + prompt_input = request.prompt if request.suffix is not None: - return self.create_error_response("suffix is not currently supported") + if request.echo: + return self.create_error_response( + "Echo is unsupported with suffix.", + param="suffix", + ) + + if request.prompt_embeds is not None: + return self.create_error_response( + "suffix is not supported with prompt_embeds", + param="suffix", + ) + + if request.truncate_prompt_tokens is not None: + return self.create_error_response( + "suffix is not supported with truncate_prompt_tokens", + param="suffix", + ) + + if isinstance(request.prompt, str): + rendered_prompt = self.renderer.render_completion_suffix( + request.prompt, request.suffix + ) + if rendered_prompt is None: + return self.create_error_response( + "suffix is only supported for models with FIM completion " + "rendering", + param="suffix", + ) + prompt_input = rendered_prompt + elif isinstance(request.prompt, list) and all( + isinstance(prompt, str) for prompt in request.prompt + ): + rendered_prompts = [] + for prompt in cast(list[str], request.prompt): + rendered_prompt = self.renderer.render_completion_suffix( + prompt, request.suffix + ) + if rendered_prompt is None: + return self.create_error_response( + "suffix is only supported for models with FIM completion " + "rendering", + param="suffix", + ) + rendered_prompts.append(rendered_prompt) + prompt_input = rendered_prompts + else: + return self.create_error_response( + "suffix requires text prompt input for FIM completion rendering", + param="suffix", + ) if request.echo and request.prompt_embeds is not None: return self.create_error_response("Echo is unsupported with prompt embeds.") @@ -562,7 +611,7 @@ class OnlineRenderer: engine_inputs = await self.preprocess_completion( request, - prompt_input=request.prompt, + prompt_input=prompt_input, prompt_embeds=request.prompt_embeds, skip_mm_cache=skip_mm_cache, ) diff --git a/vllm/scalar_type.py b/vllm/scalar_type.py index d73827f499..311b328c40 100644 --- a/vllm/scalar_type.py +++ b/vllm/scalar_type.py @@ -316,8 +316,13 @@ class ScalarType: class scalar_types: + uint2 = ScalarType.uint(2, None) + uint3 = ScalarType.uint(3, None) int4 = ScalarType.int_(4, None) uint4 = ScalarType.uint(4, None) + uint5 = ScalarType.uint(5, None) + uint6 = ScalarType.uint(6, None) + uint7 = ScalarType.uint(7, None) int8 = ScalarType.int_(8, None) uint8 = ScalarType.uint(8, None) float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN) diff --git a/vllm/transformers_utils/processors/ovis.py b/vllm/transformers_utils/processors/ovis.py index a77072745f..9ad320be39 100644 --- a/vllm/transformers_utils/processors/ovis.py +++ b/vllm/transformers_utils/processors/ovis.py @@ -110,7 +110,7 @@ class OvisProcessor(ProcessorMixin): ) -> BatchFeature: """Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode - the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to + the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwargs` arguments to Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`. Args: diff --git a/vllm/transformers_utils/processors/ovis2_5.py b/vllm/transformers_utils/processors/ovis2_5.py index c31ea91520..2d5764bfd0 100644 --- a/vllm/transformers_utils/processors/ovis2_5.py +++ b/vllm/transformers_utils/processors/ovis2_5.py @@ -115,7 +115,7 @@ class Ovis2_5Processor(ProcessorMixin): and image(s). This method forwards the `text`and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode the text. To prepare the vision inputs, - this method forwards the `vision_infos` and `kwrags` arguments to + this method forwards the `vision_infos` and `kwargs` arguments to Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`. diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index afafa3c925..a403c9ae39 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -926,6 +926,10 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): """ return False + def fused_qk_norm_mrope_kvcache_supported(self): + """Whether this implementation supports fused QKNorm+MRoPE+KVCache.""" + return False + def fused_rope_kvcache_supported(self): """Does this attention implementation support RoPE+KVCache fusion. This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops @@ -955,6 +959,26 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): """ raise NotImplementedError + def do_qk_norm_mrope_kvcache_update( + self, + layer: AttentionLayer, + qkv: torch.Tensor, + q_out: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + mrope_section: tuple[int, int, int], + is_interleaved: bool, + rotary_dim: int, + kv_cache: torch.Tensor, + layer_slot_mapping: torch.Tensor, + ): + """Apply QK-norm and MRoPE, then write K/V to the cache.""" + raise NotImplementedError + def do_rope_and_kv_cache_update( self, layer: AttentionLayer, diff --git a/vllm/v1/attention/backends/cpu_attn.py b/vllm/v1/attention/backends/cpu_attn.py index 24be5b8ec9..36df340967 100644 --- a/vllm/v1/attention/backends/cpu_attn.py +++ b/vllm/v1/attention/backends/cpu_attn.py @@ -406,6 +406,15 @@ class CPUAttentionBackendImpl(AttentionImpl): kv_cache_dtype=self.kv_cache_dtype, ) + # The CPU kernel executes attention sinks natively in bf16. If the + # sinks tensor is anything other than bf16, cast it to fp32 so it is + # executed in full float precision (done lazily here, after weights + # are loaded, rather than at __init__ time). + if self.sinks is not None and self.sinks.dtype not in [ + torch.bfloat16, + torch.float32, + ]: + self.sinks = self.sinks.to(torch.float32) ops.cpu_attention_with_kv_cache( query=query[:num_actual_tokens], key_cache=key_cache, diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index bad6ed201b..3728a451a7 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -772,9 +772,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): device=device, ) + from vllm.platforms import current_platform from vllm.v1.worker.workspace import current_workspace_manager - max_num_partial_tiles = reduce_partial_map_size + # AITER metadata sizing assumes all requests can carry max_prefill_qlen tokens, + # which is a loose worst case in the number of QO tiles. Rather, the sum of + # qlens is bounded by max_num_batched_tokens. Manually compute the number of + # qo tiles to avoid OOM at startup. + # TODO: AITER should give us this budget constrained value + qo_tile_cnt = ( + cdiv(max_num_batched_tokens, _FP8_PREFILL_TILE_Q) + max_num_reqs - 1 + ) + max_num_partial_tiles = qo_tile_cnt + current_platform.num_compute_units() reservations: list[tuple[tuple[int, ...], torch.dtype]] = [ ( (max_num_partial_tiles * _FP8_PREFILL_TILE_Q, num_head_k, v_head_dim), diff --git a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py index 9e3267c526..711a8f8755 100644 --- a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py +++ b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py @@ -7,7 +7,7 @@ from typing import ClassVar import torch from vllm import _custom_ops as ops -from vllm._aiter_ops import rocm_aiter_ops +from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops from vllm.config import get_current_vllm_config_or_none from vllm.config.cache import CacheDType from vllm.logger import init_logger @@ -359,6 +359,9 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): def fused_qk_norm_rope_kvcache_supported(self): return rocm_aiter_ops.is_enabled() + def fused_qk_norm_mrope_kvcache_supported(self) -> bool: + return IS_AITER_FOUND + def do_qk_norm_rope_kvcache_update( self, layer: AttentionLayer, @@ -397,6 +400,47 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): use_shuffle_layout=False, ) + def do_qk_norm_mrope_kvcache_update( + self, + layer: AttentionLayer, + qkv: torch.Tensor, + q_out: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + mrope_section: tuple[int, int, int], + is_interleaved: bool, + rotary_dim: int, + kv_cache: torch.Tensor, + layer_slot_mapping: torch.Tensor, + ): + key_cache, value_cache = self._split_kv_cache(kv_cache) + rocm_aiter_ops.do_qk_norm_mrope_kvcache_update( + qkv=qkv, + q_weight=q_weight, + k_weight=k_weight, + cos_sin_cache=cos_sin_cache, + positions=positions, + num_heads_q=self.num_heads, + num_heads_k=self.num_kv_heads, + head_dim=self.head_size, + is_neox=is_neox, + mrope_section=list(mrope_section), + is_interleaved=is_interleaved, + rms_norm_eps=rms_norm_eps, + q_out=q_out, + key_cache=key_cache, + value_cache=value_cache, + slot_mapping=layer_slot_mapping, + k_scale=layer._k_scale_cpu, + v_scale=layer._v_scale_cpu, + kv_cache_dtype=self.kv_cache_dtype, + rotary_dim=rotary_dim, + ) + def do_rope_and_kv_cache_update( self, layer: AttentionLayer, diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index f881fc9818..30bf584f8c 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -437,7 +437,7 @@ class KVCacheManager: ---------------------------------------------------------------------- ``` - Abbrivations: + Abbreviations: ``` comp = request.num_computed_tokens diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 66b183f192..741255a9f7 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -555,6 +555,11 @@ def _post_update_kernel( all_token_ids_ptr, all_token_ids_stride, total_len_ptr, + broadcast_drafts_ptr, + broadcast_drafts_stride, + draft_tokens_ptr, + draft_tokens_stride, + num_spec, ): req_id = tl.program_id(0) req_state_idx = tl.load(idx_mapping_ptr + req_id) @@ -562,6 +567,17 @@ def _post_update_kernel( # Filter rows with negative index entries. return + if broadcast_drafts_ptr is not None: + # PP path: adopt the draft tokens proposed by the last rank's + # speculator so the next verification step embeds the real drafts. + for i in range(num_spec): + token_id = tl.load( + broadcast_drafts_ptr + req_id * broadcast_drafts_stride + i + ) + tl.store( + draft_tokens_ptr + req_state_idx * draft_tokens_stride + i, token_id + ) + total_len = tl.load(total_len_ptr + req_state_idx) num_sampled = tl.load(num_sampled_ptr + req_id) if num_sampled > 0: @@ -622,6 +638,11 @@ def post_update( all_token_ids: torch.Tensor, # [max_num_reqs] total_len: torch.Tensor, + # [num_reqs, num_spec]; drafts broadcast from the last PP rank. Only + # passed on non-last PP ranks, which never run the speculator. + broadcast_drafts: torch.Tensor | None = None, + # [max_num_reqs, num_spec] + draft_tokens_out: torch.Tensor | None = None, ) -> None: num_reqs = idx_mapping.shape[0] _post_update_kernel[(num_reqs,)]( @@ -638,6 +659,11 @@ def post_update( all_token_ids, all_token_ids.stride(0), total_len, + broadcast_drafts, + broadcast_drafts.stride(0) if broadcast_drafts is not None else 0, + draft_tokens_out, + draft_tokens_out.stride(0) if draft_tokens_out is not None else 0, + broadcast_drafts.shape[1] if broadcast_drafts is not None else 0, num_warps=1, ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 70f601a4c0..681b29fcb9 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -860,6 +860,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # dummy run the eagle speculator's propose to ensure DP/EP sync. if self.speculator is not None: assert self.sampler is not None + assert hidden_states is not None self.step_timing.drafter_start() mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None if self.speculator.supports_mm_inputs: @@ -875,7 +876,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec_hidden_states = hidden_states if hasattr(self.model, "get_mtp_target_hidden_states"): pre_hc_hidden_states = self.model.get_mtp_target_hidden_states() - spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr] + if pre_hc_hidden_states is not None: + spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] if isinstance(self.sampler, GPUWatermarkSampler): self.speculator.prepare_watermarking( self.sampler._get_contexts(input_batch.idx_mapping), @@ -1107,6 +1109,44 @@ class GPUModelRunner(LoRAModelRunnerMixin): if outputs is not None: self.postprocess_sampled(**outputs) + def warmup_pp_decode_update(self) -> None: + """JIT-compile the kernel behind ``update_pp_decode_requests``. + + That path only runs on real steps, so the warmup steps never reach it + on non-last PP ranks. Its first triton compile must not happen + mid-serving: the in-flight sampled-token broadcast keeps a NCCL kernel + spinning on this device, which blocks the CUDA module load and + deadlocks the pipeline. An all -1 idx_mapping makes this a no-op. + The freshly allocated int32 tensors are 16-byte aligned, matching the + padded views `PPHandler` produces at serving time (triton specializes + on pointer alignment). + """ + assert self.pp_handler is not None + num_spec = self.pp_handler.max_sample_len - 1 + broadcast_drafts = ( + torch.zeros((1, num_spec), dtype=torch.int64, device=self.device) + if num_spec > 0 + else None + ) + post_update( + torch.full((1,), -1, dtype=torch.int64, device=self.device), + self.req_states.num_computed_tokens.gpu, + self.req_states.last_sampled_tokens, + None, + torch.zeros( + (1, self.pp_handler.max_sample_len), + dtype=torch.int64, + device=self.device, + ), + torch.zeros(1, dtype=torch.int32, device=self.device), + torch.zeros(1, dtype=torch.int32, device=self.device), + None, + self.req_states.all_token_ids.gpu, + self.req_states.total_len.gpu, + broadcast_drafts, + self.req_states.draft_tokens if broadcast_drafts is not None else None, + ) + def add_requests(self, scheduler_output: SchedulerOutput) -> None: for new_req_data in scheduler_output.scheduled_new_reqs: assert new_req_data.prefill_token_ids is not None @@ -1588,6 +1628,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_sampled: torch.Tensor, num_rejected: torch.Tensor, query_start_loc: torch.Tensor | None = None, + broadcast_drafts: torch.Tensor | None = None, ) -> None: # Update the number of computed tokens. output_bin_counts = None @@ -1605,6 +1646,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): query_start_loc, self.req_states.all_token_ids.gpu, self.req_states.total_len.gpu, + broadcast_drafts, + self.req_states.draft_tokens if broadcast_drafts is not None else None, ) self.model_state.postprocess_state( @@ -2125,7 +2168,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec_hidden_states = draft_hidden_states if hasattr(self.model, "get_mtp_target_hidden_states"): pre_hc_hidden_states = self.model.get_mtp_target_hidden_states() - spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)] + if pre_hc_hidden_states is not None: + spec_hidden_states = pre_hc_hidden_states[ + : draft_hidden_states.size(0) + ] if isinstance(self.sampler, GPUWatermarkSampler): self.speculator.prepare_watermarking( self.sampler._get_contexts(input_batch.idx_mapping), @@ -2147,8 +2193,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): dp_sync=dp_sync, mm_inputs=mm_inputs, ) - self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens - if self.adaptive_verification is not None: + if draft_tokens is not None: + self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens + if self.pp_handler is not None: + # Earlier stages never run the speculator; ship the + # drafts so their next verification step embeds the real + # draft tokens instead of stale buffer contents. + self.pp_handler.broadcast_drafts( + self.req_states.draft_tokens, input_batch + ) + if draft_tokens is not None and self.adaptive_verification is not None: self.adaptive_verification.record_confidences( self.speculator.draft_token_confidence_probs, input_batch ) @@ -2160,7 +2214,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_batch, self.req_states.draft_tokens[input_batch.idx_mapping], ) - if self.pp_handler is not None: + if self.pp_handler is not None and self.speculator is None: + # When a speculator ran, the propose() path above already + # broadcast the fresh drafts. Broadcasting here as well would + # double-post on the pp_broadcast group and misalign the + # recv FIFO on earlier stages, hanging the pipeline. self.pp_handler.broadcast_drafts( self.req_states.draft_tokens, input_batch ) diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index c2aaa1fba2..862b86e5a1 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -46,6 +46,18 @@ def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None: return produces_sample if produces_sample.any() else None +def _alloc_combined(num_reqs: int, device: torch.device) -> torch.Tensor: + """Allocate the (2, N) int32 buffer broadcast alongside sampled tokens. + + The inner dim is padded to a multiple of 4 so that both `unbind` views + stay 16-byte aligned for any `num_reqs`: triton specializes on pointer + alignment, and a misaligned `num_rejected` would JIT-compile a second + `_post_update_kernel` variant at serving time. The padding is broadcast + but never read. Sender and receiver must both use this allocation. + """ + return torch.empty(2, -(-num_reqs // 4) * 4, dtype=torch.int32, device=device) + + class PPHandler: """Runs the PP sampled-token broadcast/recv on a side stream so the default stream isn't gated by the matching peer call. Step T's recv is @@ -86,6 +98,14 @@ class PPHandler: ) self.aux_hidden_state_relay_keys: tuple[str, ...] = () + # Warmup steps run the pipeline with synthetic batches whose outputs are + # discarded; the sampled-token broadcast is disabled there so its + # side-stream NCCL ops cannot overlap the next step's activation p2p. + self.disabled = False + + def set_disabled(self, disabled: bool) -> None: + self.disabled = disabled + def on_req_idx_freed(self, req_idx: int) -> None: self.req_idx_gen_np[req_idx] += 1 @@ -157,29 +177,15 @@ class PPHandler: num_sampled=slot.num_sampled, num_rejected=slot.num_rejected, idx_mapping=idx_mapping, + broadcast_drafts=slot.draft_tokens, ) - def broadcast_drafts( - self, draft_tokens: torch.Tensor, input_batch: InputBatch - ) -> None: - """Broadcast draft proposals so non-last ranks can embed real token ids.""" - assert self.is_last_rank - if compute_need_sampled_mask(input_batch) is None: - return - with torch.cuda.stream(self.broadcast_stream): - self.broadcast_stream.wait_stream(self.main_stream) - send = draft_tokens[input_batch.idx_mapping].contiguous() - # Must record the idx_mapping tensor since it was allocated - # on the main stream. - input_batch.idx_mapping.record_stream(self.broadcast_stream) - torch.distributed.broadcast( - send, src=self.last_rank, group=self.broadcast_group - ) - def receive(self, input_batch: InputBatch) -> bool: """Returns True iff sampled tokens need to be gathered from *all* requests in the batch.""" assert not self.is_last_rank + if self.disabled: + return False need_sampled_mask = compute_need_sampled_mask(input_batch) if need_sampled_mask is None: # Leave this step's reserved slot as None. @@ -195,7 +201,7 @@ class PPHandler: sampled_tokens = torch.empty( num_reqs, self.max_sample_len, dtype=torch.int64, device=self.device ) - combined = torch.empty(2, num_reqs, dtype=torch.int32, device=self.device) + combined = _alloc_combined(num_reqs, self.device) torch.distributed.broadcast( sampled_tokens, src=self.last_rank, group=self.broadcast_group ) @@ -213,6 +219,7 @@ class PPHandler: torch.distributed.broadcast( draft_tokens, src=self.last_rank, group=self.broadcast_group ) + event = self.broadcast_stream.record_event() num_sampled, num_rejected = combined.unbind(dim=0) # Must record_stream since these were allocated on broadcast stream but @@ -242,11 +249,16 @@ class PPHandler: input_batch: InputBatch, ) -> None: assert self.is_last_rank - if compute_need_sampled_mask(input_batch) is None: + if self.disabled: + return + mask = compute_need_sampled_mask(input_batch) + if mask is None: # No request needs sampled outputs for a subsequent decode step. return assert sampled_token_ids.dtype == torch.int64 + assert num_sampled.dtype == torch.int32 + assert num_rejected.dtype == torch.int32 if current_platform.is_xpu(): self.main_stream.synchronize() @@ -262,9 +274,42 @@ class PPHandler: src=self.last_rank, group=self.broadcast_group, ) - combined = torch.stack((num_sampled, num_rejected), dim=0) + combined = _alloc_combined(num_sampled.shape[0], self.device) + combined[0, : num_sampled.shape[0]] = num_sampled + combined[1, : num_sampled.shape[0]] = num_rejected torch.distributed.broadcast( combined, src=self.last_rank, group=self.broadcast_group ) for tensor in (sampled_token_ids, num_sampled, num_rejected): tensor.record_stream(self.broadcast_stream) + + def broadcast_drafts( + self, draft_token_table: torch.Tensor, input_batch: InputBatch + ) -> None: + """Broadcast the speculator's freshly proposed draft tokens. + + Runs after propose() on the last rank; the send is stream-ordered + after broadcast()'s sends, matching receive()'s enqueue order. The + payload is gathered from the runner's draft table (just updated from + propose()'s output) into a fresh compact tensor: the speculator + overwrites its own persistent buffer on the next step, possibly + before this async send completes. + """ + assert self.is_last_rank + if self.disabled or self.max_sample_len == 1: + return + if compute_need_sampled_mask(input_batch) is None: + return + assert draft_token_table.dtype == torch.int64 + assert draft_token_table.shape[1] == self.max_sample_len - 1 + + # Gather on the main stream so the payload is ordered after this + # step's table update and before any later one. + drafts = draft_token_table[input_batch.idx_mapping] + assert drafts.shape[0] == input_batch.num_reqs + with torch.cuda.stream(self.broadcast_stream): + self.broadcast_stream.wait_stream(self.main_stream) + torch.distributed.broadcast( + drafts, src=self.last_rank, group=self.broadcast_group + ) + drafts.record_stream(self.broadcast_stream) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index a5710829e0..3f661bbda5 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -256,6 +256,7 @@ class DFlashSpeculator(DraftModelSpeculator): num_tokens_across_dp=num_tokens_across_dp, slot_mapping=slot_mappings, batch_descriptor=batch_descriptor, + is_padding=self.input_buffers.is_padding[:num_tokens], ): last_hidden_states = self.model( input_ids=self.input_buffers.input_ids[:num_tokens], @@ -487,6 +488,7 @@ def _prepare_dflash_inputs_kernel( # Outputs out_input_ids_ptr, out_query_positions_ptr, + out_is_padding_ptr, out_query_start_loc_ptr, out_seq_lens_ptr, out_query_slot_mapping_ptr, @@ -617,6 +619,7 @@ def _prepare_dflash_inputs_kernel( ) tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query) + tl.store(out_is_padding_ptr + query_idx, False, mask=is_query) clamped_query_pos = tl.minimum(query_pos, max_model_len - 1) tl.store(out_query_positions_ptr + query_idx, clamped_query_pos, mask=is_query) tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query) @@ -679,6 +682,9 @@ def _prepare_dflash_inputs_kernel( for i in range(q_pad_start, max_num_tokens, BLOCK_SIZE): block = i + tl.arange(0, BLOCK_SIZE) mask = block < max_num_tokens + tl.store(out_input_ids_ptr + block, 0, mask=mask) + tl.store(out_query_positions_ptr + block, 0, mask=mask) + tl.store(out_is_padding_ptr + block, True, mask=mask) tl.store(out_query_slot_mapping_ptr + block, PAD_SLOT_ID, mask=mask) @@ -730,6 +736,7 @@ def prepare_dflash_inputs( _prepare_dflash_inputs_kernel[(num_reqs, num_blocks)]( input_buffers.input_ids, input_buffers.positions, + input_buffers.is_padding, input_buffers.query_start_loc, input_buffers.seq_lens, query_slot_mapping, diff --git a/vllm/v1/worker/gpu/warmup.py b/vllm/v1/worker/gpu/warmup.py index 56f7d438fc..2b7f6b6d49 100644 --- a/vllm/v1/worker/gpu/warmup.py +++ b/vllm/v1/worker/gpu/warmup.py @@ -445,6 +445,11 @@ def _warmup_kernels( for step_indices, step_spec_flags in decode_steps: _run_decode_step(step_indices, step_spec_flags) + # The deferred PP post-update path only runs on real steps, so the steps + # above never JIT-compile its kernel on non-last ranks. + if not model_runner.is_last_pp_rank and model_runner.pp_handler is not None: + model_runner.warmup_pp_decode_update() + # Clean up - process finish_req_ids. cleanup_output = SchedulerOutput.make_empty() cleanup_output.finished_req_ids = set(req_ids) diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 3d4c5f3d5c..58af821127 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -805,6 +805,15 @@ class Worker(WorkerBase): @instrument(span_name="Warmup (GPU)") def compile_or_warm_up_model(self) -> CompilationTimes: + # All warmup phases below run synthetic steps whose sampled outputs are + # discarded. The PP sampled-token broadcast would carry no payload, and + # its side-stream NCCL ops can overlap the next step's activation p2p + # and deadlock the pipeline, so keep it disabled for the whole warmup + # window and restore it before serving. + pp_handler = getattr(self.model_runner, "pp_handler", None) + if pp_handler is not None: + pp_handler.set_disabled(True) + warmup_sizes: list[int] = [] if ( @@ -983,6 +992,9 @@ class Worker(WorkerBase): # gate so subsequent `execute_model` / `sample_tokens` calls enforce it. enable_gpu_sync_check() + if pp_handler is not None: + pp_handler.set_disabled(False) + return CompilationTimes( language_model=self.compilation_config.compilation_time, encoder=self.compilation_config.encoder_compilation_time,