From b978df3299518ed7b02584f69d6daac55eab0b71 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Thu, 17 Sep 2026 14:31:59 +0800 Subject: [PATCH 001/143] Revert "[Frontend] Omit absent fields from /inference/v1/generate stream chunks" (#57302) --- .../server/src/routes/inference/generate.rs | 30 ---------------- .../test_generate_stream.py | 34 ------------------- .../scale_out/token_in_token_out/serving.py | 10 ++++-- 3 files changed, 7 insertions(+), 67 deletions(-) diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 2ce42d8c33..dfb7e6cbb1 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -679,36 +679,6 @@ mod tests { assert!(response.mm_placeholders.is_none()); } - #[test] - fn stream_chunk_omits_absent_fields() { - // Contract shared with the Python frontend: absent optional fields - // are omitted from stream chunks, not serialized as null. - let chunk = GenerateStreamResponse { - request_id: "raw-1".to_string(), - choices: vec![GenerateResponseStreamChoice { - index: 0, - logprobs: None, - finish_reason: None, - token_ids: vec![1], - }], - usage: None, - prompt_token_ids: None, - mm_placeholders: None, - }; - - let json = serde_json::to_value(&chunk).expect("serialize chunk"); - - for key in ["usage", "prompt_token_ids", "mm_placeholders"] { - assert!(json.get(key).is_none(), "{key} should be omitted"); - } - for key in ["logprobs", "finish_reason"] { - assert!( - json["choices"][0].get(key).is_none(), - "{key} should be omitted" - ); - } - } - #[test] fn extract_mm_placeholders_returns_none_without_features() { assert!(extract_mm_placeholders(None).is_none()); diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index 0cb8a6d397..d9ae424cc8 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -569,40 +569,6 @@ async def test_stream_omits_prompt_metadata_by_default(): assert "mm_placeholders" not in chunk -@pytest.mark.asyncio -async def test_stream_chunks_omit_absent_fields(): - """Absent optional fields are omitted from stream chunks, like Rust.""" - engine = _mock_engine() - - async def mock_generate(*args, **kwargs): - yield _make_request_output("req-1", token_ids=[10]) - yield _make_request_output( - "req-1", token_ids=[20], finish_reason="stop", finished=True - ) - - engine.generate = MagicMock(side_effect=mock_generate) - serving = _build_serving_tokens(engine) - request = GenerateRequest( - token_ids=[1, 2, 3], - sampling_params=SamplingParams(max_tokens=2), - model=MODEL_NAME, - stream=True, - ) - - response = await serving.serve_tokens(request) - parsed = _parse_sse_chunks([chunk async for chunk in response]) - data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")] - - assert len(data_chunks) == 2 - for chunk in data_chunks: - assert "usage" not in chunk - assert "prompt_token_ids" not in chunk - assert "mm_placeholders" not in chunk - assert "logprobs" not in chunk["choices"][0] - assert "finish_reason" not in data_chunks[0]["choices"][0] - assert data_chunks[1]["choices"][0]["finish_reason"] == "stop" - - @pytest.mark.asyncio async def test_stream_text_only_prompt_omits_mm_placeholders_key(): engine = _mock_engine() diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 838fe90c5c..a8a2804a06 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -486,9 +486,13 @@ class ServingTokens(GenerateBaseServing): total_tokens=(num_prompt_tokens + num_generated_tokens[i]), ) - # Omit absent fields, like the Rust frontend and the - # final usage chunk below. - yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" + # Omit absent prompt metadata (Rust skips None fields). + exclude = { + name + for name in ("prompt_token_ids", "mm_placeholders") + if getattr(chunk, name) is None + } + yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n" total_completion_tokens = sum(num_generated_tokens) final_usage_info = UsageInfo( From fdcd42350bd2475abffc01bb73eec5d7c7a1e549 Mon Sep 17 00:00:00 2001 From: Shanshan Shen <467638484@qq.com> Date: Thu, 17 Sep 2026 14:46:00 +0800 Subject: [PATCH 002/143] [ROCm][Perf] Enable HCA dual-stream overlap for DeepSeek-V4 (#56853) Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> --- vllm/models/deepseek_v4/amd/rocm.py | 44 +++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index f8ac39cd62..545f29bd35 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -558,14 +558,17 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): self._fused_compressor_split_sizes: tuple[int, int] | None = None if self.indexer is None: - # Only enable multi-stream overlap for CSA layer now. - self.aux_stream_list = None + # Dense layers have no compressor work to overlap; HCA layers + # (compressor, no indexer) keep the streams for the dual-stream + # fork below. + if self.compressor is None: + self.aux_stream_list = None else: # Disable indexer inner overlap. self.indexer.aux_stream = None - def _enable_csa_multi_stream(self) -> bool: - """All CSA multi-stream gates: env var, streams, and capture region. + def _enable_multi_stream_overlap(self) -> bool: + """ROCm multi-stream gates: streams and capture region. Dict metadata marks piecewise cudagraph, whose eager breaks rebuild the attention inputs on the owning stream. Forking side streams @@ -573,7 +576,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): this overlap on ROCm (event waits can hang), so multi-stream only runs where the fork/join becomes static graph edges: inside capture, or with non-dict metadata (full cudagraph or the profile run), which - has no eager breaks. + has no eager breaks. Covers both the HCA and CSA forks. """ attn_metadata = get_forward_context().attn_metadata return self.aux_stream_list is not None and ( @@ -624,7 +627,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): device=hidden_states.device, ) - if self._enable_csa_multi_stream(): + if self._enable_multi_stream_overlap(): # The ROCm override consumes these sentinels inside the capture # boundary, moving the stream fan-out ahead of the projections. self._prepare_and_attn_fn( @@ -658,7 +661,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): positions: torch.Tensor, o_padded: torch.Tensor, ) -> None: - """Run the ROCm CSA fork/join inside the graph capture boundary.""" + """Run the ROCm fork/join (HCA or CSA) inside the capture boundary.""" aux_streams = self.aux_stream_list # The sequential pipeline disables aux_stream_list before calling # back with real projection inputs; aux_streams is None ends that @@ -684,13 +687,13 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): # Re-check: forward's gate ran inside a captured segment that # _prepare_and_attn_eager (MRV1) then broke, making this region eager. - if not self._enable_csa_multi_stream(): + if not self._enable_multi_stream_overlap(): self._run_sequential_pipeline(hidden_states, positions, o_padded) return indexer = self.indexer compressor = self.compressor - assert indexer is not None and compressor is not None + assert compressor is not None def default_chain(): qr_kv = self._fused_wqa_wkv_gemm(hidden_states) @@ -710,6 +713,24 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): ) compressor(score, positions, self.rotary_emb) + if indexer is None: + # HCA dual-stream: the main compressor runs on aux stream 0 while + # the default stream produces q and inserts KV into the SWA cache. + # Both branches only read hidden_states, so the join merely has to + # precede the sparse attention that consumes the compressed KV. + (q, _qr_out, _qr_scale_out, kv_out), _ = execute_in_parallel( + default_chain, + [main_compressor_chain], + self.ln_events[0], + [self.ln_events[1]], + aux_streams[:1], + enable=True, + ) + self._sparse_indexer_and_attn( + hidden_states, None, None, None, q, kv_out, positions, o_padded + ) + return + def indexer_compressor_chain() -> None: score = torch.mm( hidden_states, @@ -718,6 +739,11 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): ) indexer.compressor(score, positions, self.indexer_rotary_emb) + # CSA three-stream: the main and indexer compressors run on aux + # streams 0 and 1 while the default stream produces q and inserts KV + # into the SWA cache. Every branch only reads hidden_states plus its + # own state, so the join merely has to precede the indexer op and the + # sparse attention, which consume the compressed KV caches. (q, qr_out, qr_scale_out, kv_out), _ = execute_in_parallel( default_chain, [main_compressor_chain, indexer_compressor_chain], From bd2d7ae7c197ba995800bb03a23b4d79c38627a4 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 17 Sep 2026 03:01:19 -0400 Subject: [PATCH 003/143] [Perf] Reuse MoE workspace for DeepGEMM warmup (#57268) Signed-off-by: Lucas Wilkinson --- vllm/model_executor/warmup/deep_gemm_warmup.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/warmup/deep_gemm_warmup.py b/vllm/model_executor/warmup/deep_gemm_warmup.py index 6aa5d391e0..1266f4ccb2 100644 --- a/vllm/model_executor/warmup/deep_gemm_warmup.py +++ b/vllm/model_executor/warmup/deep_gemm_warmup.py @@ -30,6 +30,7 @@ from vllm.utils.deep_gemm import ( ) from vllm.utils.math_utils import cdiv from vllm.utils.platform_utils import num_compute_units +from vllm.v1.worker.workspace import current_workspace_manager def _generate_optimal_warmup_m_values( @@ -275,15 +276,15 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup( ) if not warmup_cases: return - device = w1.device def _warmup(w: torch.Tensor, w_scale: torch.Tensor): _, n, k = w.size() - a1q = torch.empty((MAX_M, k), device=device, dtype=torch.float8_e4m3fn) - a1q_scales = torch.zeros( - (MAX_M, k // block_m), device=device, dtype=torch.float32 + a1q, a1q_scales, out = current_workspace_manager().get_simultaneous( + ((MAX_M, k), torch.float8_e4m3fn), + ((MAX_M, k // block_m), torch.float32), + ((MAX_M, n), torch.bfloat16), ) - out = torch.empty((MAX_M, n), device=device, dtype=torch.bfloat16) + a1q_scales.zero_() for num_tokens, align_used, expert_ids in warmup_cases: with mk_alignment_scope(align_used): From 7bbce752b85f183eb34e27ea6df9647f80986060 Mon Sep 17 00:00:00 2001 From: Shuolei Wang <46160365+ShuoleiWang@users.noreply.github.com> Date: Thu, 17 Sep 2026 15:02:24 +0800 Subject: [PATCH 004/143] [Perf][Model] Qwen4Exp QSA: sm_90 tuning table for _select_config (#57273) Signed-off-by: Shuolei Wang Signed-off-by: Shuolei Wang <948904026@qq.com> Co-authored-by: Roger Wang --- vllm/models/qwen4_exp/nvidia/ops/qsa.py | 40 ++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa.py b/vllm/models/qwen4_exp/nvidia/ops/qsa.py index f0aa62a484..816ccdd498 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa.py @@ -22,6 +22,12 @@ def _is_sm120() -> bool: return current_platform.get_device_capability() == (12, 0) +@lru_cache(maxsize=1) +def _is_sm90() -> bool: + """True on sm_90 (H100/H200/H20): selects the sm_90 tuning table.""" + return current_platform.get_device_capability() == (9, 0) + + @triton.jit(do_not_specialize=["num_rows", "num_requests"]) def _qsa_sparse_paged_gqa_splitk_kernel( q_ptr, @@ -509,6 +515,33 @@ def _select_sm120_config( return 32, 4, 1 +def _select_sm90_config( + base_programs: int, use_prefill_config: bool, is_fp8: bool +) -> tuple[int, int, int]: + """(block_n, target_splits, num_warps) tuned on sm_90 (H20).""" + if base_programs > 2048: + return (32, 1, 1) if use_prefill_config else (64, 1, 2) + if is_fp8: + if base_programs <= 24: + return 64, 64, 2 + if base_programs <= 32: + return 32, 16, 1 + if base_programs <= 64: + return 32, 8, 1 + if base_programs <= 128: + return 32, 4, 1 + if base_programs <= 256: + return 32, 8, 1 + return 32, 4, 1 + if base_programs <= 32: + return 64, 64, 2 + if base_programs <= 64: + return 32, 16, 1 + if base_programs <= 256: + return 32, 8, 1 + return 32, 4, 1 + + def _select_config( num_rows: int, num_kv_heads: int, @@ -521,13 +554,18 @@ def _select_config( Keyed on base_programs = num_rows * num_kv_heads. The bp > 2048 region splits on use_prefill_config (capture-stable: at FULL-graph capture max_query_len is the uniform decode/verify length). This default table was tuned on GB300; - sm_120 (RTX PRO 6000 Blackwell) dispatches to _select_sm120_config instead. + sm_120 (RTX PRO 6000 Blackwell) dispatches to _select_sm120_config instead, + and sm_90 (Hopper) to _select_sm90_config. """ base_programs = num_rows * num_kv_heads if _is_sm120(): BLOCK_N, target_splits, num_warps = _select_sm120_config( base_programs, use_prefill_config, is_fp8 ) + elif _is_sm90(): + BLOCK_N, target_splits, num_warps = _select_sm90_config( + base_programs, use_prefill_config, is_fp8 + ) elif base_programs > 2048: BLOCK_N, target_splits, num_warps = ( (32, 1, 1) if use_prefill_config else (64, 1, 2) From b3079e6e46a21109a8641c6d48641453a9c7cb86 Mon Sep 17 00:00:00 2001 From: leo <92263211+liuyao0322@users.noreply.github.com> Date: Thu, 17 Sep 2026 15:04:53 +0800 Subject: [PATCH 005/143] [Perf][DSpark] Stack DeepSeek V4 context WKV projections (#54674) Signed-off-by: liuyao0322 Co-authored-by: OpenAI Codex --- tests/models/test_dspark_mla.py | 73 ++++++++++++++++++++++++ vllm/models/deepseek_v4/nvidia/dspark.py | 52 +++++++++++++++-- 2 files changed, 119 insertions(+), 6 deletions(-) diff --git a/tests/models/test_dspark_mla.py b/tests/models/test_dspark_mla.py index d30eb9e7c9..3e70450f2c 100644 --- a/tests/models/test_dspark_mla.py +++ b/tests/models/test_dspark_mla.py @@ -11,6 +11,7 @@ from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead from vllm.model_executor.models.registry import ModelRegistry +from vllm.models.deepseek_v4.nvidia import dspark as dsv4_dspark from vllm.models.kimi_k3.nvidia import dspark_mla from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel @@ -262,3 +263,75 @@ def test_v41_dspark_loads_linear_scales( assert loaded == {runtime_name} assert shards == [() if shard_id is None else (shard_id,)] torch.testing.assert_close(param, checkpoint_scale) + + +def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer(): + weights = [ + ("mtp.0.attn.wkv.weight", torch.arange(4)), + ("mtp.1.attn.wq_a.weight", torch.arange(3)), + ("mtp.2.attn.wkv.scale", torch.tensor(0.5)), + ("mtp.3.attn.wkv.weight", torch.arange(2)), + ] + + duplicated = list(dsv4_dspark._duplicate_context_wkv_weights(weights, 3)) + + assert [name for name, _ in duplicated] == [ + "mtp.0.attn.wkv.weight", + "context_wkv_proj.weight", + "mtp.1.attn.wq_a.weight", + "mtp.2.attn.wkv.scale", + "context_wkv_proj.scale", + "mtp.3.attn.wkv.weight", + ] + assert duplicated[1][1].shard_id == 0 + assert duplicated[4][1].shard_id == 2 + assert duplicated[0][1].data_ptr() == duplicated[1][1].data_ptr() + assert duplicated[3][1].data_ptr() == duplicated[4][1].data_ptr() + + +def test_dsv4_context_kv_uses_one_stacked_wkv_projection(monkeypatch): + calls = [] + stacked_output = torch.arange(24, dtype=torch.float32).view(2, 12) + + class StackedProjection: + def __init__(self): + self.calls = 0 + + def __call__(self, main_x): + self.calls += 1 + assert main_x.shape == (2, 5) + return stacked_output + + projection = StackedProjection() + layers = [ + SimpleNamespace(attn=SimpleNamespace(kv_norm=lambda kv, offset=i: kv + offset)) + for i in range(3) + ] + model = SimpleNamespace( + config=SimpleNamespace(head_dim=4), + context_wkv_proj=projection, + layers=layers, + num_dspark_layers=3, + ) + slot_mappings = [torch.tensor([0, 1]), None, torch.tensor([4, 5])] + monkeypatch.setattr( + dsv4_dspark, + "_insert_context_kv", + lambda attn, kv, positions, slots: calls.append( + (attn, kv.clone(), positions, slots) + ), + ) + + dsv4_dspark.DSparkDeepseekV4Model.precompute_and_store_context_kv( + model, + torch.zeros(2, 5), + torch.tensor([7, 8]), + slot_mappings, + ) + + assert projection.calls == 1 + assert len(calls) == 2 + assert torch.equal(calls[0][1], stacked_output.view(2, 3, 4)[:, 0]) + assert torch.equal(calls[1][1], stacked_output.view(2, 3, 4)[:, 2] + 2) + assert calls[0][3] is slot_mappings[0] + assert calls[1][3] is slot_mappings[2] diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index 1a3c2c780d..52783889d6 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -31,7 +31,10 @@ from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + ReplicatedLinear, +) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -61,6 +64,24 @@ logger = init_logger(__name__) # MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders): # fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``. _EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$") +_CONTEXT_WKV_RE = re.compile(r"^mtp\.(\d+)\.attn\.wkv\.(.+)$") + + +def _duplicate_context_wkv_weights( + weights: Iterable[tuple[str, torch.Tensor]], num_layers: int +) -> Iterable[tuple[str, torch.Tensor]]: + """Load every draft layer's WKV into the cross-layer projection.""" + for name, weight in weights: + yield name, weight + match = _CONTEXT_WKV_RE.fullmatch(name) + if match is None: + continue + layer_idx = int(match.group(1)) + if layer_idx >= num_layers: + continue + stacked_weight = weight.detach() + stacked_weight.shard_id = layer_idx + yield f"context_wkv_proj.{match.group(2)}", stacked_weight class DSparkDeepseekV4Model(nn.Module): @@ -113,6 +134,15 @@ class DSparkDeepseekV4Model(nn.Module): for i in range(self.num_dspark_layers) ] ) + self.context_wkv_proj = MergedColumnParallelLinear( + config.hidden_size, + [config.head_dim] * self.num_dspark_layers, + bias=False, + return_bias=False, + quant_config=vllm_config.quant_config, + prefix=maybe_prefix(prefix, "context_wkv_proj"), + disable_tp=True, + ) # Heads: final norm + hc_head, and the Markov + confidence heads # Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint @@ -173,15 +203,16 @@ class DSparkDeepseekV4Model(nn.Module): place draft layers in different groups). ``None`` (or a ``None`` entry) runs the projection to reserve workspace but writes nothing (profiling). """ - for i, layer in enumerate(self.layers): + all_kv = self.context_wkv_proj(main_x).view( + main_x.shape[0], self.num_dspark_layers, self.config.head_dim + ) + for i, (layer, kv) in enumerate( + zip(self.layers, all_kv.unbind(1), strict=True) + ): slot_mapping = ( None if context_slot_mappings is None else context_slot_mappings[i] ) attn = layer.attn - # Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection - # (q_lora part discarded), then RoPE/quant/insert via the fused op. - qr_kv, _ = attn.fused_wqa_wkv(main_x) - kv = qr_kv[..., attn.q_lora_rank :] kv = attn.kv_norm(kv) if slot_mapping is None: continue @@ -429,6 +460,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): head_start = n_local_head * tp_rank head_end = n_local_head * (tp_rank + 1) + weights = _duplicate_context_wkv_weights(weights, len(self.model.layers)) for name, loaded_weight in weights: mapped = self._remap_dspark_name(name) if mapped is None: @@ -452,6 +484,12 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): self.quant_config, name, loaded_weight ) + if name.startswith("model.context_wkv_proj."): + param = params_dict[name] + param.weight_loader(param, loaded_weight, loaded_weight.shard_id) + loaded_params.add(name) + continue + # E8M0 expert scales: keep raw exponent bytes. if ".experts." in name: if ( @@ -522,6 +560,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): Returns None for non-mtp weights (owned by the target model). """ + if name.startswith("context_wkv_proj."): + return f"model.{name}" m = re.match(r"mtp\.(\d+)\.(.*)", name) if m is None: return None From 528fa835fd37d0805a824d516e22f88df61a771a Mon Sep 17 00:00:00 2001 From: JohnQinAMD Date: Thu, 17 Sep 2026 00:14:43 -0700 Subject: [PATCH 006/143] [ROCm] Stage large pageable H2D copies instead of registering them (#56343) Signed-off-by: JohnQinAMD Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Shanshan Shen <467638484@qq.com> --- vllm/platforms/rocm.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 248f41d20d..9089d025e0 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -21,6 +21,7 @@ from .interface import DeviceCapability, Platform, PlatformEnum, in_wsl if TYPE_CHECKING: from vllm.config import VllmConfig from vllm.config.kernel import IrOpPriorityConfig + from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.v1.attention.selector import AttentionSelectorConfig logger = init_logger(__name__) @@ -493,6 +494,9 @@ class RocmPlatform(Platform): dist_backend: str = "nccl" # rocm shares the same device control env var as CUDA device_control_env_var: str = "CUDA_VISIBLE_DEVICES" + # Set in pre_register_and_update, so it exists only on the driver; Ray + # workers are separate processes and copy env vars by allowlist. + additional_env_vars: list[str] = ["GPU_PINNED_MIN_XFER_SIZE"] ray_noset_device_env_vars: list[str] = [ "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES", "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", @@ -862,6 +866,16 @@ class RocmPlatform(Platform): ) return torch.cuda.get_device_properties(device_id).total_memory + @classmethod + def pre_register_and_update( + cls, parser: "FlexibleArgumentParser | None" = None + ) -> None: + # Keep mmap'd weight pages on the HIP staging path: above this + # threshold the runtime registers the pageable source instead, and each + # registration's MMU notifier makes KFD suspend our queues. In KB, so + # 4 GiB. + os.environ.setdefault("GPU_PINNED_MIN_XFER_SIZE", str(4 * 1024 * 1024)) + @classmethod def apply_config_platform_defaults(cls, vllm_config: "VllmConfig") -> None: from vllm._aiter_ops import rocm_aiter_ops From 4e5ffda19d897f3c34455167dc0503620f1b3e2a Mon Sep 17 00:00:00 2001 From: "Jiahan Chang (Cyrus)" <173873397+jiahanc@users.noreply.github.com> Date: Thu, 17 Sep 2026 15:38:58 +0800 Subject: [PATCH 007/143] [Bugfix] Isolate supplemental FlashInfer BF16 autotuning (#57285) Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: OpenAI Codex --- vllm/model_executor/warmup/kernel_warmup.py | 43 +++++++++++++++++++-- vllm/utils/flashinfer.py | 43 +++++++++++++++++---- 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index bf2c1ad0b6..81eca7ac50 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -330,13 +330,16 @@ def _flashinfer_deferred_moe_token_counts( return tuple(dict.fromkeys(token_counts)) -def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]: +def _flashinfer_autotune_token_counts( + runner: "GPUModelRunner", *, include_bf16: bool = True +) -> tuple[int, ...]: max_tokens = runner.scheduler_config.max_num_batched_tokens # Tune the widest bucket set first so bounded passes reuse its configs. token_counts = [max_tokens] linear_backend = runner.vllm_config.kernel_config.linear_backend if ( - linear_backend == "flashinfer_cutedsl" + include_bf16 + and linear_backend == "flashinfer_cutedsl" and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS ): token_counts.append(_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS) @@ -350,7 +353,7 @@ def _run_flashinfer_autotune_dummy_runs( import vllm.utils.flashinfer as fi_utils dummy_run_kwargs = {"skip_attn": True} if skip_attn else {} - for num_tokens in _flashinfer_autotune_token_counts(runner): + for num_tokens in _flashinfer_autotune_token_counts(runner, include_bf16=False): tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens) logger.info( "Running FlashInfer autotune with %d tokens and token buckets %s.", @@ -367,6 +370,36 @@ def _run_flashinfer_autotune_dummy_runs( ) +def _run_flashinfer_bf16_autotune_dummy_run( + runner: "GPUModelRunner", + *, + skip_ops: set[str] | None = None, + skip_attn: bool = False, +) -> None: + import vllm.utils.flashinfer as fi_utils + + if ( + runner.vllm_config.kernel_config.linear_backend != "flashinfer_cutedsl" + or runner.scheduler_config.max_num_batched_tokens + <= _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS + ): + return + + num_tokens = _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS + tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens) + logger.info("Running FlashInfer BF16-only autotune with %d tokens.", num_tokens) + # Other ops execute from their existing cache or fallback, without the + # BF16 bucket cap. No full-model autotune context may enclose this pass. + with fi_utils.autotune_bf16_only(tuning_buckets, skip_ops=skip_ops): + runner._dummy_run( + num_tokens=num_tokens, + skip_eplb=True, + is_profile=True, + randomize_inputs=True, + **({"skip_attn": True} if skip_attn else {}), + ) + + def flashinfer_autotune(runner: "GPUModelRunner") -> None: """Autotune FlashInfer operations. FlashInfer have many implementations for the same operation, @@ -435,6 +468,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: _run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled) replayssm_autotune_warmup(runner) _autotune_kimi_k3_kda_qkvg(runner.get_model()) + with torch.inference_mode(): + _run_flashinfer_bf16_autotune_dummy_run( + runner, skip_ops=skip_ops, skip_attn=hisparse_enabled + ) finally: set_autotune_process_group(None) diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index 7647179134..abc4d75500 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -11,7 +11,8 @@ import importlib import importlib.util import os import shutil -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextvars import ContextVar from typing import Any, NoReturn import requests @@ -26,6 +27,24 @@ from vllm.utils.torch_utils import PIN_MEMORY logger = init_logger(__name__) +_bf16_autotune_buckets: ContextVar[tuple[int, ...] | None] = ContextVar( + "flashinfer_bf16_autotune_buckets", default=None +) + + +@contextlib.contextmanager +def autotune_bf16_only( + tuning_buckets: tuple[int, ...], *, skip_ops: set[str] | None = None +) -> Iterator[None]: + """Tune BF16 calls with bounded buckets, outside full-model autotuning.""" + token = _bf16_autotune_buckets.set(tuning_buckets) + try: + with autotune(tune_mode=False, skip_ops=skip_ops): + yield + finally: + _bf16_autotune_buckets.reset(token) + + def flashinfer_bf16_mm( a: torch.Tensor, b: torch.Tensor, @@ -35,14 +54,21 @@ def flashinfer_bf16_mm( ) -> torch.Tensor: from flashinfer import mm_bf16 - return mm_bf16( - a, - b, - bias=bias, - pdl=pdl, - out_dtype=torch.bfloat16, - backend=backend, + tuning_buckets = _bf16_autotune_buckets.get() + tuning = ( + autotune(tune_mode=True, tuning_buckets=tuning_buckets) + if tuning_buckets is not None + else contextlib.nullcontext() ) + with tuning: + return mm_bf16( + a, + b, + bias=bias, + pdl=pdl, + out_dtype=torch.bfloat16, + backend=backend, + ) # This is the storage path for the cubins, it can be replaced @@ -1237,6 +1263,7 @@ def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool: __all__ = [ "has_flashinfer", "flashinfer_bf16_mm", + "autotune_bf16_only", "has_flashinfer_bf16_gemm", "is_flashinfer_bf16_gemm_supported", "is_flashinfer_cutedsl_bf16_gemm_supported", From de24e5190820cc6640f2d55295405f228fffbc41 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Thu, 17 Sep 2026 16:12:17 +0800 Subject: [PATCH 008/143] [MM][V2] Enable encoder-only ViT CUDA graph capture (#56922) Signed-off-by: jiangkuaixue123 Co-authored-by: Codex --- .../test_areas/disaggregated_mooncake.yaml | 10 ++ .../generation/test_vit_cudagraph.py | 115 ++++++++++++++++++ tests/v1/cudagraph/test_encoder_cudagraph.py | 95 +++++++++++++++ .../run_epd_mooncake_ec_full_pipeline.sh | 36 +++++- .../integration/test_epd_correctness.py | 21 ++++ tests/v1/worker/test_encoder_runner.py | 55 ++++++++- vllm/v1/worker/mm_encoder_model_runner.py | 35 +++++- 7 files changed, 362 insertions(+), 5 deletions(-) diff --git a/.buildkite/test_areas/disaggregated_mooncake.yaml b/.buildkite/test_areas/disaggregated_mooncake.yaml index 8c061a95ad..f6e944d705 100644 --- a/.buildkite/test_areas/disaggregated_mooncake.yaml +++ b/.buildkite/test_areas/disaggregated_mooncake.yaml @@ -40,7 +40,14 @@ steps: - vllm/v1/worker/gpu_worker.py - vllm/v1/worker/gpu/ec_connector.py - vllm/v1/worker/gpu/model_runner.py + - vllm/v1/worker/mm_encoder_model_runner.py + - vllm/v1/worker/encoder_cudagraph.py + - vllm/v1/worker/encoder_cudagraph_defs.py + - vllm/v1/attention/ops/vit_attn_wrappers.py + - vllm/v1/worker/gpu/model_states/ - vllm/v1/worker/gpu/mm/ + - tests/v1/cudagraph/test_encoder_cudagraph.py + - tests/models/multimodal/generation/test_vit_cudagraph.py - examples/disaggregated/disaggregated_encoder/ - tests/v1/ec_connector/ - requirements/kv_connectors.txt @@ -49,6 +56,9 @@ steps: PYTHONUNBUFFERED: "1" VLLM_HOST_IP: "127.0.0.1" VLLM_USE_V2_MODEL_RUNNER: "1" + E_CUDAGRAPH_MM_ENCODER: "1" + # FA padding NaNs are tracked separately in #57136. + E_MM_ENCODER_ATTN_BACKEND: "TRITON_ATTN" MOONCAKE_EC_PROTOCOL: "tcp" USE_MM_PROMPTS: "1" SKIP_BASELINE: "0" diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index f8136ae8f3..1ed84ebc52 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -499,3 +499,118 @@ def test_vit_cudagraph_video(model_id, vllm_runner, video_assets): # Ensure the output is a string assert isinstance(output_text, str) + + +def _check_eonly_encoder_outputs(worker, batches): + """Compare real E-only replay with eager, retaining outputs across replay.""" + import torch + + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + + runner = worker.model_runner + assert isinstance(runner, MMEncoderModelRunner) + manager = runner.model_state.encoder_runner.cudagraph_manager + assert manager is not None and manager.is_captured() + retained: list[tuple[torch.Tensor, torch.Tensor]] = [] + max_error = 0.0 + stats = [] + with torch.inference_mode(): + for batch in batches: + kwargs = {key: value.to(runner.device) for key, value in batch.items()} + actual = manager.execute(kwargs) + expected = runner.model.embed_multimodal(**kwargs) + assert len(actual) == len(expected) + for output, reference in zip(actual, expected): + assert torch.isfinite(output).all() + # Two BF16 ulps at unit scale, fixed before model validation. + torch.testing.assert_close(output, reference, rtol=0.016, atol=0.016) + max_error = max(max_error, (output - reference).abs().max().item()) + for output, snapshot in retained: + torch.testing.assert_close(output, snapshot, rtol=0, atol=0) + retained.extend((output, output.clone()) for output in actual) + stats.append(manager.get_cumulative_stats()) + assert stats[0]["graph_hits"] > 0 + assert stats[1]["graph_hits"] > stats[0]["graph_hits"] + assert stats[2]["graph_misses"] > stats[1]["graph_misses"] + assert stats[3]["graph_hits"] > stats[2]["graph_hits"] + assert stats[4]["graph_hits"] > stats[3]["graph_hits"] + assert stats[5]["graph_hits"] > stats[4]["graph_hits"] + assert stats[6]["graph_hits"] > stats[5]["graph_hits"] + return {"max_abs_error": max_error, "stats": stats} + + +@pytest.mark.parametrize( + "model_id", + params_with_marks({key: MODEL_CONFIGS[key] for key in ("qwen3_vl", "qwen3_5")}), +) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA graphs") +def test_eonly_vit_cudagraph_outputs( + model_id, vllm_runner, image_assets, video_assets, monkeypatch +): + """Exercise startup capture, image/video replay, fallback and output ownership.""" + import numpy as np + from PIL import Image + from transformers import AutoProcessor + + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + config = MODEL_CONFIGS[model_id] + token_budgets = [64, 256, 2048] + processor = AutoProcessor.from_pretrained(config.model) + first, second = [asset.pil_image for asset in image_assets] + batches = [ + [first.resize((224, 224))], + [second.resize((448, 224)), first.resize((224, 224))], + [second.resize((1792, 1792))], + [second.resize((224, 224))], + [first.resize((1280, 720))], + [first.resize((224, 224))], + ] + frames = sample_frames_from_video(video_assets[0].np_ndarrays, 2) + video = np.stack( + [np.asarray(Image.fromarray(frame).resize((224, 224))) for frame in frames] + ) + with vllm_runner( + config.model, + dtype=config.dtype, + mm_encoder_only=True, + # FA padding NaNs are tracked separately in #57136. + mm_encoder_attn_backend="FLASHINFER", + enable_prefix_caching=False, + max_model_len=4096, + max_num_seqs=2, + limit_mm_per_prompt={"image": 2, "video": 1}, + compilation_config={ + "cudagraph_mode": "NONE", + "cudagraph_mm_encoder": True, + "encoder_cudagraph_token_budgets": token_budgets, + "encoder_cudagraph_max_vision_items_per_batch": 2, + "encoder_cudagraph_max_frames_per_batch": 2, + }, + ) as model: + mm_config = model.llm.llm_engine.vllm_config.model_config.multimodal_config + processor_kwargs = mm_config.merge_mm_processor_kwargs({}) + inputs = [ + dict( + processor.image_processor( + images=images, return_tensors="pt", **processor_kwargs + ) + ) + for images in batches + ] + inputs.append( + dict( + processor.video_processor( + videos=[video], + do_sample_frames=False, + return_tensors="pt", + **processor_kwargs, + ) + ) + ) + results = model.llm.collective_rpc( + partial(_check_eonly_encoder_outputs, batches=inputs) + ) + for result in results: + assert result["stats"][0]["token_budgets"] == token_budgets + print(f"E-only {model_id}: {results}") diff --git a/tests/v1/cudagraph/test_encoder_cudagraph.py b/tests/v1/cudagraph/test_encoder_cudagraph.py index 096f0e4075..1f145bdc89 100644 --- a/tests/v1/cudagraph/test_encoder_cudagraph.py +++ b/tests/v1/cudagraph/test_encoder_cudagraph.py @@ -13,7 +13,9 @@ Test organization: """ from collections.abc import Hashable +from types import SimpleNamespace from typing import Any +from unittest.mock import MagicMock, patch import pytest import torch @@ -575,6 +577,99 @@ class TestEncoderCudaGraphCaptureReplay: assert out.shape == (4, _HIDDEN) +# --------------------------------------------------------------------------- +# E-only capture and output lifecycle +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm" +) +@pytest.mark.usefixtures("dist_init", "workspace_init") +@pytest.mark.parametrize("profile_only", [False, True]) +@torch.inference_mode() +def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only): + """The E-only entry captures only the encoder and preserves cached outputs.""" + from vllm.distributed.ec_transfer.ec_connector.base import ( + ECConnectorBase, + ECConnectorMetadata, + ) + from vllm.v1.worker.gpu.ec_connector import ActiveECConnector + from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + from vllm.v1.worker.workspace import lock_workspace + + device = torch.device("cuda:0") + dtype = torch.float16 + model = SimpleMockViTModel().to(device).half() + manager = _make_manager_for_gpu(model, _BUDGETS, _MAX_BATCH, device, dtype) + encoder = object.__new__(EncoderRunner) + encoder.device = device + encoder.cudagraph_manager = manager + runner = object.__new__(MMEncoderModelRunner) + runner.model_state = SimpleNamespace(encoder_runner=encoder) + # No decoder manager is installed: capture must be encoder-only. + with patch( + "vllm.v1.worker.mm_encoder_model_runner.lock_workspace", wraps=lock_workspace + ) as lock: + runner.capture_model(profile_only=profile_only) + assert lock.call_count == int(not profile_only) + assert len(manager.budget_graphs["default"]) == len(_BUDGETS) + + cache: dict[str, torch.Tensor] = {} + pending_sends: dict[str, torch.Tensor] = {} + connector = MagicMock(spec=ECConnectorBase) + connector.is_producer = True + connector.is_consumer = False + connector.get_finished.return_value = (None, None) + connector.save_caches.side_effect = lambda *, encoder_cache, mm_hash: ( + pending_sends.update({mm_hash: encoder_cache[mm_hash]}) + ) + with patch( + "vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector + ): + ec = ActiveECConnector(SimpleNamespace(), cache) + scheduled = SimpleNamespace( + ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset() + ) + saved_outputs: list[tuple[torch.Tensor, torch.Tensor]] = [] + # Mixed sizes exercise packing order; 64 is the boundary, 81 falls back. + for grids in ([[1, 8, 8], [1, 4, 4]], [[1, 16, 16]], [[1, 18, 18]], [[1, 4, 4]]): + inputs = _make_mm_kwargs(grids, device, dtype) + expected = model.encoder_eager_forward(inputs).split( + [t * (h // 2) * (w // 2) for t, h, w in grids] + ) + with ec.maybe_get_output(scheduled) as ec_output: + outputs = manager.execute(inputs) + assert outputs is not None + cache[str(len(cache))] = outputs[0] + assert ec_output.finished_sending is None + assert outputs is not None + for actual, eager in zip(outputs, expected): + torch.testing.assert_close(actual, eager) + for previous, snapshot in saved_outputs: + torch.testing.assert_close(previous, snapshot, rtol=0, atol=0) + saved_outputs.extend((output, output.clone()) for output in outputs) + assert manager.graph_hits == 4 + assert manager.graph_misses == 1 + assert pending_sends.keys() == cache.keys() + connector.get_finished.return_value = (set(cache), None) + with ec.maybe_get_output(scheduled) as ec_output: + pass + assert ec_output.finished_sending == set(cache) + + +def test_eonly_without_encoder_graph_skips_capture(): + from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + + encoder = object.__new__(EncoderRunner) + encoder.cudagraph_manager = None + runner = object.__new__(MMEncoderModelRunner) + runner.model_state = SimpleNamespace(encoder_runner=encoder) + assert runner.capture_model() == 0 + + # --------------------------------------------------------------------------- # SimpleMockViTVideoModel — extends SimpleMockViTModel with video support # --------------------------------------------------------------------------- diff --git a/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh b/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh index 05af1bb6fd..c4e9b0d069 100755 --- a/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh +++ b/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh @@ -20,6 +20,8 @@ # SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE # CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2) # MAX_MODEL_LEN context length (default 16384) +# E_CUDAGRAPH_MM_ENCODER enable encoder CUDA graphs (default 0) +# E_MM_ENCODER_ATTN_BACKEND optional encoder attention backend override set -euo pipefail @@ -35,6 +37,15 @@ if [[ "$USE_MM_PROMPTS" == "1" ]]; then TEST_ARGS+=(--use_mm_prompts --mm_smoke_test) fi MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}" +E_ATTN_ARGS=() +if [[ -n "${E_MM_ENCODER_ATTN_BACKEND:-}" ]]; then + E_ATTN_ARGS+=(--mm-encoder-attn-backend "$E_MM_ENCODER_ATTN_BACKEND") +fi +case "${E_CUDAGRAPH_MM_ENCODER:-0}" in + 0) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":false}' ;; + 1) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":true}' ;; + *) echo "E_CUDAGRAPH_MM_ENCODER must be 0 or 1"; exit 1 ;; +esac GPU_SINGLE="${GPU_SINGLE:-0}" GPU_E="${GPU_E:-0}" @@ -203,7 +214,7 @@ run_epd_mooncake() { cleanup_instances echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT" - CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \ + VLLM_SERVER_DEV_MODE=1 CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \ --port "$ENCODE_PORT" \ --gpu-memory-utilization 0.35 \ --mm-tensor-ipc torch_shm \ @@ -215,6 +226,9 @@ run_epd_mooncake() { --limit-mm-per-prompt '{"image":2,"video":0}' \ --allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \ --ec-transfer-config "$ENC_EC_JSON" \ + --compilation-config "$E_COMPILATION_CONFIG" \ + "${E_ATTN_ARGS[@]}" \ + --worker-extension-cls tests.v1.ec_connector.integration.test_epd_correctness.EncoderGraphTestWorkerExtension \ >"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 & local ENCODER_PID=$! PIDS+=("$ENCODER_PID") @@ -266,6 +280,26 @@ run_epd_mooncake() { --baseline_file "$BASELINE_FILE" \ "${TEST_ARGS[@]}" + curl -fsS "http://localhost:${ENCODE_PORT}/collective_rpc" \ + -H 'Content-Type: application/json' \ + -d '{"method":"encoder_graph_stats"}' \ + >"${LOG_PATH}/encoder_graph_stats.json" + "$PYTHON_BIN" - "${LOG_PATH}/encoder_graph_stats.json" "${E_CUDAGRAPH_MM_ENCODER:-0}" <<'PY' +import json +import sys + +with open(sys.argv[1]) as f: + results = json.load(f)["results"] +assert results, "No encoder workers returned graph statistics" +for stats in results: + if sys.argv[2] == "1": + assert stats["is_captured"], stats + assert stats["graph_hits"] > 0, stats + else: + assert stats["graph_hits"] == 0, stats +print("Encoder graph statistics:", results) +PY + cleanup_instances } diff --git a/tests/v1/ec_connector/integration/test_epd_correctness.py b/tests/v1/ec_connector/integration/test_epd_correctness.py index b8565a0acd..4e37637ea4 100644 --- a/tests/v1/ec_connector/integration/test_epd_correctness.py +++ b/tests/v1/ec_connector/integration/test_epd_correctness.py @@ -26,6 +26,7 @@ import json import os import time from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING import openai import requests @@ -33,6 +34,26 @@ import requests from vllm.assets.image import ImageAsset from vllm.multimodal.utils import encode_image_url +if TYPE_CHECKING: + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + + +class EncoderGraphTestWorkerExtension: + """Expose encoder replay evidence through the existing test RPC endpoint.""" + + model_runner: "MMEncoderModelRunner" + + def encoder_graph_stats(self): + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + + runner = self.model_runner + assert isinstance(runner, MMEncoderModelRunner) + manager = runner.model_state.encoder_runner.cudagraph_manager + if manager is None: + return {"is_captured": False, "graph_hits": 0, "graph_misses": 0} + return {"is_captured": manager.is_captured(), **manager.get_cumulative_stats()} + + MAX_OUTPUT_LEN = 256 # Sample prompts with multimodal content diff --git a/tests/v1/worker/test_encoder_runner.py b/tests/v1/worker/test_encoder_runner.py index f8bd50c6d9..98aaa568f0 100644 --- a/tests/v1/worker/test_encoder_runner.py +++ b/tests/v1/worker/test_encoder_runner.py @@ -9,12 +9,15 @@ and tolerated (token-embedding fallback) when it is not, while a miss within the processed range still fails loudly. """ -from unittest.mock import MagicMock +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import numpy as np import pytest import torch +from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph from vllm.multimodal.inputs import ( MultiModalFeatureSpec, MultiModalFieldElem, @@ -31,6 +34,56 @@ pytestmark = pytest.mark.cpu_test HIDDEN = 4 +@pytest.mark.parametrize( + "enabled,enforce_eager,supported,expected", + [ + (True, False, True, True), + (False, False, True, False), + (True, True, True, False), + (True, False, False, False), + ], +) +def test_encoder_graph_configuration(enabled, enforce_eager, supported, expected): + """Encoder graph eligibility is independent of decoder graph mode.""" + config = MagicMock() + config.model_config.enforce_eager = enforce_eager + config.model_config.dtype = torch.float32 + config.model_config.get_inputs_embeds_size.return_value = HIDDEN + config.scheduler_config.max_num_batched_tokens = 8 + config.compilation_config.cudagraph_mode = CUDAGraphMode.NONE + config.compilation_config.cudagraph_mm_encoder = enabled + model = MagicMock(spec=SupportsEncoderCudaGraph) if supported else torch.nn.Module() + state = _model_state(EncoderCache()) + with patch("vllm.v1.worker.gpu.model_states.interface.EncoderCudaGraphManager"): + ModelState.__init__(state, config, model, state.encoder_cache, state.device) + assert state.encoder_runner.has_cudagraph() is expected + + +def test_encoder_graph_unsupported_modality_uses_eager_output(): + """An image graph must not intercept another modality's encoder output.""" + from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager + + runner = _make_runner([], []) + output = torch.ones(2, HIDDEN) + runner.model = MagicMock() + runner.model.embed_multimodal.return_value = [output] + manager = MagicMock(spec=EncoderCudaGraphManager) + manager.config = SimpleNamespace(modalities=["image"]) + manager.is_captured.return_value = True + manager.supports_modality.side_effect = lambda modality: ( + EncoderCudaGraphManager.supports_modality(manager, modality) + ) + runner.cudagraph_manager = manager + with patch( + "vllm.v1.worker.gpu.mm.encoder_runner.group_and_batch_mm_kwargs", + return_value=[("audio", 1, {"audio_values": torch.zeros(2)})], + ): + result = runner.execute_mm_encoder([]) + assert len(result) == 1 + assert result[0] is output + manager.execute.assert_not_called() + + def _model_state(cache: EncoderCache) -> MagicMock: """A mock ModelState backed by a real EncoderCache.""" state = MagicMock() diff --git a/vllm/v1/worker/mm_encoder_model_runner.py b/vllm/v1/worker/mm_encoder_model_runner.py index e19f1b332a..4dc0bf77dd 100644 --- a/vllm/v1/worker/mm_encoder_model_runner.py +++ b/vllm/v1/worker/mm_encoder_model_runner.py @@ -4,9 +4,10 @@ An encoder-only instance -- `--mm-encoder-only`, or the producer side of encoder-cache disaggregation -- encodes the multi-modal items and publishes the -embeddings. It runs no language model: no KV cache, no sampler, no CUDA graphs. +embeddings. It runs no language model: no KV cache, sampler, or decoder graphs. """ +import time from collections.abc import Iterator from contextlib import contextmanager from typing import TYPE_CHECKING, Any @@ -15,7 +16,9 @@ import torch from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.logger import init_logger from vllm.sequence import IntermediateTensors +from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture from vllm.v1.kv_cache_interface import KVCacheSpec from vllm.v1.outputs import ( ModelRunnerOutput, @@ -24,10 +27,13 @@ from vllm.v1.outputs import ( from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras from vllm.v1.worker.gpu.model_runner import GPUModelRunner +from vllm.v1.worker.workspace import lock_workspace if TYPE_CHECKING: from vllm.v1.core.sched.output import SchedulerOutput +logger = init_logger(__name__) + class MMEncoderModelRunner(GPUModelRunner): """Encoder-only variant of the V2 GPU model runner.""" @@ -67,8 +73,31 @@ class MMEncoderModelRunner(GPUModelRunner): def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]: return {} + @torch.inference_mode() def capture_model(self, *, profile_only: bool = False) -> int: - return 0 + encoder_runner = self.model_state.encoder_runner + if not encoder_runner.has_cudagraph(): + return 0 + + start_time = time.perf_counter() + with freeze_gc_for_cudagraph_capture(): + torch.accelerator.empty_cache() + start_free_memory = torch.accelerator.get_memory_info()[0] + encoder_runner.capture() + graph_memory = start_free_memory - torch.accelerator.get_memory_info()[0] + + if not profile_only: + lock_workspace() + + manager = encoder_runner.cudagraph_manager + assert manager is not None + logger.info( + "Encoder graph capturing finished in %.0f secs, took %.2f GiB (%d graphs)", + time.perf_counter() - start_time, + graph_memory / (1 << 30), + manager.get_num_graphs_to_capture(), + ) + return graph_memory def needs_cudagraph_capture(self) -> bool: return False @@ -114,7 +143,7 @@ class MMEncoderModelRunner(GPUModelRunner): batch_req_state, _ = self.gather_batch_req_state(scheduler_output, False) assert batch_req_state is not None - # No CUDA graph, and no DP peer to agree a padded shape with. + # No decoder graph or DP padding; encoder graphs run independently. batch_desc = BatchExecutionDescriptor( cg_mode=CUDAGraphMode.NONE, num_tokens=batch_req_state.num_tokens, From 88afb777008bab6d33a6f5ea965e2a0d83a0a7ec Mon Sep 17 00:00:00 2001 From: coderfornow <39123181+coderfornow@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:05:17 +0530 Subject: [PATCH 009/143] [CPU][s390x] Pin protobuf to 7.36.1 and drop C++ extension removal workaround (#54978) Signed-off-by: coderfornow Co-authored-by: Li, Jiang --- docker/Dockerfile.s390x | 6 ------ docs/getting_started/installation/cpu.s390x.inc.md | 13 ------------- requirements/common.txt | 3 ++- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/docker/Dockerfile.s390x b/docker/Dockerfile.s390x index 9517ff3718..6faba540a9 100644 --- a/docker/Dockerfile.s390x +++ b/docker/Dockerfile.s390x @@ -169,8 +169,6 @@ ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-14/root/usr/lib64/pkgconfig:/usr/local/ ENV PATH="${VIRTUAL_ENV:+${VIRTUAL_ENV}/bin}:/opt/rh/gcc-toolset-14/root/usr/bin:/usr/local/bin:$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH" ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} -# Force pure Python protobuf to avoid s390x C++ extension crashes -ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python COPY . /workspace/vllm WORKDIR /workspace/vllm @@ -214,10 +212,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \ uv pip install "$(echo dist/*.whl)[tensorizer]" -# Remove protobuf C++ extension that crashes on s390x -RUN rm -rf /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/_upb/*.so \ - /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/protobuf/pyext/*.so 2>/dev/null || true - # setup non-root user for vllm RUN umask 002 && \ /usr/sbin/useradd --uid 2000 --gid 0 vllm && \ diff --git a/docs/getting_started/installation/cpu.s390x.inc.md b/docs/getting_started/installation/cpu.s390x.inc.md index 88767e4c6e..e1ff4ee3cb 100644 --- a/docs/getting_started/installation/cpu.s390x.inc.md +++ b/docs/getting_started/installation/cpu.s390x.inc.md @@ -138,19 +138,6 @@ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \ The Docker image (`Dockerfile.s390x`) already includes TCMalloc and sets `LD_PRELOAD` automatically. -!!! warning "Protobuf workaround for s390x" - The C++ protobuf extension crashes on s390x. After installation, set the - following environment variable and remove the C++ extensions: - - ```bash - export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python - - # Remove C++ protobuf extensions that crash on s390x - SITE_PKGS=$(python -c "import site; print(site.getsitepackages()[0])") - rm -rf "$SITE_PKGS/google/_upb/"*.so \ - "$SITE_PKGS/google/protobuf/pyext/"*.so 2>/dev/null || true - ``` - --8<-- [end:build-wheel-from-source] --8<-- [start:pre-built-images] diff --git a/requirements/common.txt b/requirements/common.txt index acfc986c75..ff691ebe47 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -11,7 +11,8 @@ transformers >= 5.10.4 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export tokenizers >= 0.21.1 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 -protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.* # Required by LlamaTokenizer, gRPC. CVE-2026-0994 +protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.*; platform_machine != "s390x" +protobuf == 7.36.1; platform_machine == "s390x" fastapi[standard] >= 0.133.0, < 0.137.0 # First version supporting Starlette 1.0; < 0.137.0 avoids route-tree change that breaks model-hosting-container-standards handler overrides. starlette >= 1.0.1 # CVE-2026-48710: Host header injection in < 1.0.1 aiohttp >= 3.13.3 From 23cd0346cec1cb98862d93e0c1bfa490f827e6a3 Mon Sep 17 00:00:00 2001 From: Tianmu Li Date: Thu, 17 Sep 2026 01:43:08 -0700 Subject: [PATCH 010/143] [CPU] Align scheduler and NIXL CPU affinity per local rank (#53636) Signed-off-by: Li, Tianmu Co-authored-by: OpenAI Codex Co-authored-by: Li, Jiang --- docs/getting_started/installation/cpu.md | 4 ++-- vllm/platforms/cpu.py | 3 ++- vllm/utils/ompmultiprocessing.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md index 539c28d709..53b8eb596d 100644 --- a/docs/getting_started/installation/cpu.md +++ b/docs/getting_started/installation/cpu.md @@ -150,7 +150,7 @@ VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cpu VLLM_TARGET_DEVICE=cpu - `VLLM_CPU_KVCACHE_SPACE`: specify the KV Cache size (e.g, `VLLM_CPU_KVCACHE_SPACE=40` means 40 GiB space for KV cache), larger setting will allow vLLM to run more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users. Default value is `0`. - `VLLM_CPU_OMP_THREADS_BIND`: specify the CPU cores dedicated to the OpenMP threads, can be set as CPU id lists, `auto` (by default), or `nobind` (to disable binding to individual CPU cores and to inherit user-defined OpenMP variables). For example, `VLLM_CPU_OMP_THREADS_BIND=0-31` means there will be 32 OpenMP threads bound on 0-31 CPU cores. `VLLM_CPU_OMP_THREADS_BIND=0-31|32-63` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores. By setting to `auto`, the OpenMP threads of each rank are bound to the CPU cores in each NUMA node respectively. If set to `nobind`, the number of OpenMP threads is determined by the standard `OMP_NUM_THREADS` environment variable. -- `VLLM_CPU_NUM_OF_RESERVED_CPU`: specify the number of CPU cores which are not dedicated to the OpenMP threads for each rank. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. Default value is `None`. If the value is not set and use `auto` thread binding, no CPU will be reserved for `world_size == 1`, 1 CPU per rank will be reserved for `world_size > 1`. +- `VLLM_CPU_NUM_OF_RESERVED_CPU`: specify the total number of CPU cores which are not dedicated to OpenMP threads. The variable only takes effect when `VLLM_CPU_OMP_THREADS_BIND` is set to `auto`. By default, x86, ARM, and RISC-V reserve `local_world_size` CPUs in total, while PowerPC and S390X retain a one-CPU default. When KV transfer is enabled, one additional CPU is reserved per local rank. Reserved CPUs are distributed across the local ranks. - `CPU_VISIBLE_MEMORY_NODES`: specify visible NUMA memory nodes for vLLM CPU workers, similar to ```CUDA_VISIBLE_DEVICES```. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. The variable provides more control for the auto thread-binding feature, such as masking nodes and changing nodes binding sequence. - `VLLM_ZENTORCH_WEIGHT_PREPACK` (AMD Zen only): when `ZenCpuPlatform` is active, eagerly prepack linear weights into ZenDNN's blocked layout at model load time, eliminating per-inference layout conversion overhead. Default is `1` (enabled). See [AMD Zen optimizations](#amd-zen-optimizations). @@ -253,7 +253,7 @@ See [AMD Zen optimizations](#amd-zen-optimizations) for detection rules, support ### How to decide `VLLM_CPU_OMP_THREADS_BIND`? -- Default `auto` thread-binding is recommended for most cases. Ideally, each OpenMP thread will be bound to a dedicated physical core respectively, threads of each rank will be bound to the same NUMA node respectively, and 1 CPU per rank will be reserved for other vLLM components when `world_size > 1`. If you have any performance problems or unexpected binding behaviours, please try to bind threads as following. +- Default `auto` thread-binding is recommended for most cases. Ideally, each OpenMP thread will be bound to a dedicated physical core respectively, threads of each rank will be bound to the same NUMA node respectively, and the default reserved CPUs will be kept for other vLLM components. On x86, ARM, and RISC-V, `local_world_size` CPUs are reserved in total, plus one additional CPU per local rank when KV transfer is enabled. If you have any performance problems or unexpected binding behaviours, please try to bind threads as following. - On a hyper-threading enabled platform with 16 logical CPU cores / 8 physical CPU cores: diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 6cc1475b44..390c0dbfed 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -515,6 +515,7 @@ class CpuPlatform(Platform): if not (os.path.exists(SYS_NODE) and os.path.exists(SYS_CPU)): return [] + use_highest_sibling = cls.get_cpu_architecture() == CpuArchEnum.X86 core_rsv_for_kv = [] for node in os.listdir(SYS_NODE): if not node.startswith("node") or not node[4:].isdigit(): @@ -547,7 +548,7 @@ class CpuPlatform(Platform): else: siblings = [cpu_id] - phys = min(siblings) + phys = max(siblings) if use_highest_sibling else min(siblings) if phys not in seen_phys: seen_phys.add(phys) diff --git a/vllm/utils/ompmultiprocessing.py b/vllm/utils/ompmultiprocessing.py index 57f0016b22..34395489fb 100644 --- a/vllm/utils/ompmultiprocessing.py +++ b/vllm/utils/ompmultiprocessing.py @@ -46,12 +46,12 @@ class OMPProcessManager: self.reserve_cpu_num = ( self.local_world_size if current_platform.get_cpu_architecture() - in (CpuArchEnum.ARM, CpuArchEnum.RISCV) + in (CpuArchEnum.X86, CpuArchEnum.ARM, CpuArchEnum.RISCV) else 1 ) # reserve at one more core for nixl_connector under p/d case if config.kv_transfer_config: - self.reserve_cpu_num += 1 + self.reserve_cpu_num += self.local_world_size if envs.VLLM_CPU_NUM_OF_RESERVED_CPU is not None: if self.reserve_cpu_num > envs.VLLM_CPU_NUM_OF_RESERVED_CPU: From f1c2f6ada80f7c0d896b7870f53db97103ff339d Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 17 Sep 2026 03:44:11 -0500 Subject: [PATCH 011/143] [ROCm][CI] Stage H gating and MI355 test reallocation (#57080) Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex Co-authored-by: OpenAI Codex --- .buildkite/test-amd.yaml | 1648 ++++++++++++----- .buildkite/test_areas/compile.yaml | 4 +- .buildkite/test_areas/cuda.yaml | 4 +- .buildkite/test_areas/disaggregated.yaml | 14 +- .buildkite/test_areas/distributed.yaml | 58 +- .buildkite/test_areas/e2e_integration.yaml | 2 +- .buildkite/test_areas/engine.yaml | 4 + .buildkite/test_areas/entrypoints.yaml | 24 +- .buildkite/test_areas/kernels.yaml | 189 +- .buildkite/test_areas/lm_eval.yaml | 28 +- .buildkite/test_areas/lora.yaml | 18 +- .buildkite/test_areas/misc.yaml | 43 +- .buildkite/test_areas/models_basic.yaml | 21 +- .buildkite/test_areas/models_distributed.yaml | 8 +- .buildkite/test_areas/models_language.yaml | 13 +- .buildkite/test_areas/models_multimodal.yaml | 7 +- .buildkite/test_areas/plugins.yaml | 14 + .buildkite/test_areas/pytorch.yaml | 16 + .buildkite/test_areas/quantization.yaml | 44 + .buildkite/test_areas/ray_compat.yaml | 16 + .buildkite/test_areas/rust_frontend.yaml | 4 +- .buildkite/test_areas/spec_decode.yaml | 42 +- .buildkite/test_areas/weight_loading.yaml | 2 +- 23 files changed, 1686 insertions(+), 537 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 4b72256a3f..d038d66176 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -124,8 +124,9 @@ steps: - tests/models/test_vision.py - tests/models/test_adapters.py - tests/models/transformers/fusers/ + - tests/models/test_qwen3_5_mtp_config.py commands: - - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/transformers/fusers/ + - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ - label: ":computer: (CPU) Multimodal Processor Shard %N" # TBD timeout_in_minutes: 90 @@ -214,7 +215,7 @@ steps: - .buildkite/test_areas/rust_frontend_cargo.yaml - .buildkite/scripts/run-rust-frontend-cargo-ci.sh commands: - - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy + - .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy - label: ":computer: (CPU) Rust Frontend Cargo" # TBD timeout_in_minutes: 35 @@ -229,7 +230,7 @@ steps: - .buildkite/test_areas/rust_frontend_cargo.yaml - .buildkite/scripts/run-rust-frontend-cargo-ci.sh commands: - - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh test + - .buildkite/scripts/run-rust-frontend-cargo-ci.sh test - label: ":computer: (CPU) Docker Build Metadata" # TBD timeout_in_minutes: 35 @@ -281,7 +282,7 @@ steps: #-------------------------------------------------------- mi250 · distributed --------------------------------------------------------# - label: ":amd: (MI250) Pipeline + Context Parallelism" # TBD - timeout_in_minutes: 80 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_4 num_gpus: 4 @@ -297,14 +298,18 @@ steps: - tests/distributed/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/v1/distributed/test_pp_dp_v2.py + optional: true commands: - pytest -v -s distributed/test_pp_cudagraph.py - pytest -v -s distributed/test_pipeline_parallel.py + - pytest -v -s v1/distributed/test_pp_dp_v2.py #---------------------------------------------------------- mi250 · kernels ----------------------------------------------------------# -- label: ":amd: (MI250) Helion Kernels" # TBD - timeout_in_minutes: 40 +- label: ":amd: (MI250) Helion Kernels Shard %N" + timeout_in_minutes: 45 + parallelism: 2 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 optional: true @@ -316,25 +321,43 @@ steps: - vllm/platforms/rocm.py commands: - pip install helion==1.4.0 - - pytest -v -s kernels/helion/ + - pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT #----------------------------------------------------- mi250 · models / language -----------------------------------------------------# - label: ":amd: (MI250) Language Models (PPL)" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ - tests/models/language/generation_ppl_test + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/models/qwen3_5_mtp.py + - vllm/transformers_utils/configs/qwen3_5.py + - vllm/transformers_utils/configs/qwen3_5_moe.py + - vllm/model_executor/models/qwen2.py + - vllm/model_executor/models/qwen3.py + - vllm/model_executor/models/qwen3_next.py + - vllm/model_executor/models/qwen3_next_mtp.py + - vllm/model_executor/layers/fla/ops/ + - vllm/_aiter_ops.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/backends/rocm_attn.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/v1/attention/backends/flex_attention.py + - vllm/v1/attention/ops/ + - vllm/platforms/rocm.py + optional: true commands: - pytest -v -s models/language/generation_ppl_test #---------------------------------------------------- mi250 · models / multimodal ----------------------------------------------------# - label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 optional: true @@ -367,7 +390,7 @@ steps: - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py - label: ":amd: (MI250) CUDAGraph" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 optional: true @@ -378,9 +401,13 @@ steps: - vllm/config/compilation.py - vllm/compilation - vllm/platforms/rocm.py + - vllm/v1/worker/encoder_cudagraph.py + - vllm/v1/worker/encoder_cudagraph_defs.py commands: - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py - pytest -v -s v1/cudagraph/test_cudagraph_mode.py + - pytest -v -s v1/cudagraph/test_breakable_cudagraph.py + - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py - label: ":amd: (MI250) E2E Core" # TBD timeout_in_minutes: 50 @@ -423,6 +450,7 @@ steps: - vllm/engine/arg_utils.py - vllm/transformers_utils/config.py - vllm/platforms/rocm.py + - vllm/v1/engine/ commands: - pytest -v -s v1/engine/test_preprocess_error_handling.py - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py @@ -447,7 +475,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/draft_model/ - label: ":amd: (MI250) Spec Decode Speculators + MTP" # TBD - timeout_in_minutes: 65 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 optional: true @@ -461,6 +489,7 @@ steps: - vllm/transformers_utils/configs/speculators/ - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py + - vllm/v1/attention/backends/ commands: - pytest -v -s v1/e2e/spec_decode/speculators/ - pytest -v -s v1/e2e/spec_decode/mtp/ @@ -468,7 +497,7 @@ steps: #------------------------------------------------------------- mi250 · misc ------------------------------------------------------------# - label: ":amd: (MI300) Python-only Installation" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -501,14 +530,16 @@ steps: - tests/basic_correctness/test_basic_correctness - tests/basic_correctness/test_cpu_offload - tests/basic_correctness/test_mem.py + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s basic_correctness/test_mem.py - VLLM_TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py - pytest -v -s basic_correctness/test_cpu_offload.py -- label: ":amd: (MI300) Distributed Models" # TBD +- label: ":amd: (MI300) Distributed Models Shard %N" timeout_in_minutes: 80 + parallelism: 3 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -531,21 +562,17 @@ steps: - tests/basic_correctness/ - tests/model_executor/model_loader/test_sharded_state_loader.py - tests/models/ + - vllm/v1/worker/ + - tests/conftest.py + - tests/utils.py + - tests/models/test_vision.py commands: - - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - - HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' - - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' - - pytest models/language -v -s -m 'distributed(num_gpus=2)' - - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py - - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' - - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' - - pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)' - - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' + - case "$$BUILDKITE_PARALLEL_JOB" in 0) TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' && HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' ;; 1) pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' && pytest models/language -v -s -m 'distributed(num_gpus=2)' && VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py && pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' && pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; esac #-------------------------------------------------------- mi300 · benchmarks ---------------------------------------------------------# - label: ":amd: (MI300) Benchmarks CLI" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -554,13 +581,14 @@ steps: source_file_dependencies: - vllm/ - tests/benchmarks/ + - rust/src/bench/tests/python_serve_flags.txt commands: - pytest -v -s benchmarks/ #---------------------------------------------------------- mi300 · compile ----------------------------------------------------------# - label: ":amd: (MI300) PyTorch Compilation" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -576,6 +604,29 @@ steps: - csrc/ - tests/compile - vllm/platforms/rocm.py + - vllm/__init__.py + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/env_override.py + - vllm/envs.py + - vllm/forward_context.py + - vllm/inputs/ + - vllm/ir/ + - vllm/kernels/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/plugins/ + - vllm/sampling_params.py + - vllm/sequence.py + - vllm/transformers_utils/ + - vllm/triton_utils/ + - vllm/utils/ + - vllm/v1/ commands: - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" @@ -596,6 +647,7 @@ steps: - tests/compile/fusions_e2e/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - rocm-smi - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3" @@ -615,6 +667,7 @@ steps: - tests/compile/fusions_e2e/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - rocm-smi # Run all models and attn backends but only Inductor partition and native custom ops @@ -631,11 +684,37 @@ steps: source_file_dependencies: - vllm/ - tests/compile/passes + - vllm/platforms/rocm.py + - vllm/__init__.py + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/compilation/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/env_override.py + - vllm/envs.py + - vllm/forward_context.py + - vllm/inputs/ + - vllm/ir/ + - vllm/kernels/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/plugins/ + - vllm/sampling_params.py + - vllm/sequence.py + - vllm/transformers_utils/ + - vllm/triton_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: - pytest -s -v compile/passes --ignore compile/passes/distributed - label: ":amd: (MI300) PyTorch Fullgraph" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -649,11 +728,33 @@ steps: - csrc/ - tests/compile - vllm/platforms/rocm.py + - vllm/__init__.py + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/env_override.py + - vllm/envs.py + - vllm/forward_context.py + - vllm/inputs/ + - vllm/ir/ + - vllm/kernels/ + - vllm/logger.py + - vllm/multimodal/ + - vllm/platforms/ + - vllm/plugins/ + - vllm/sampling_params.py + - vllm/sequence.py + - vllm/transformers_utils/ + - vllm/triton_utils/ + - vllm/utils/ + - vllm/v1/ commands: - - pytest -v -s compile/fullgraph/test_full_graph.py + - find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}' - label: ":amd: (MI300) PyTorch Nightly Dependency Override Check" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -704,7 +805,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "dynamo_partition" - label: ":amd: (MI300) Distributed Compile + RPC" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -722,9 +823,9 @@ steps: - tests/compile/test_wrapper.py - tests/entrypoints/llm/test_collective_rpc.py - vllm/platforms/rocm.py + optional: true commands: - pytest -v -s entrypoints/llm/test_collective_rpc.py - - pytest -v -s ./compile/fullgraph/test_basic_correctness.py - pytest -v -s ./compile/test_wrapper.py #----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------# @@ -738,9 +839,16 @@ steps: source_file_dependencies: - vllm/ - tests/cuda + - vllm/envs.py + - vllm/logger.py + - vllm/platforms/ + - vllm/plugins/ + - vllm/utils/ + optional: true commands: - pytest -v -s cuda/test_cuda_context.py - pytest -v -s cuda/test_platform_no_cuda_init.py + - pytest -v -s cuda/test_cuda_compatibility_path.py #-------------------------------------------------------- mi300 · detokenizer --------------------------------------------------------# @@ -756,6 +864,19 @@ steps: - tests/detokenizer - tests/multimodal - tests/utils_ + - vllm/assets/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: - pytest -v -s detokenizer - pytest -v -s -m 'not cpu_test' multimodal @@ -782,7 +903,7 @@ steps: - pytest -v -s distributed/test_shm_storage.py - label: ":amd: (MI300) EPLB Algorithm" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -814,7 +935,7 @@ steps: - pytest -v -s distributed/test_eplb_spec_decode.py - label: ":amd: (MI300) Distributed Features" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -831,6 +952,26 @@ steps: - examples/features/data_parallel/data_parallel_offline.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - csrc/custom_quickreduce.cu + - csrc/ops.h + - csrc/torch_bindings.cpp + - vllm/model_executor/layers/ + - vllm/entrypoints/llm.py + - vllm/config/parallel.py + - vllm/v1/engine/ + - vllm/v1/executor/ + - vllm/v1/worker/ + - vllm/_custom_ops.py + - vllm/envs.py + - tests/distributed/test_rocm_aiter_custom_ar.py + - tests/distributed/test_rocm_quick_reduce.py + - tests/distributed/test_quick_all_reduce.py + - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py + - tests/utils.py + - examples/rl/rlhf_async_new_apis.py + - tests/distributed/test_weight_transfer.py + - tests/distributed/test_packed_tensor.py + optional: true commands: - pytest -v -s tests/distributed/test_context_parallel.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py @@ -839,6 +980,10 @@ steps: - pytest -v -s tests/v1/distributed/test_dbo.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py - pytest -v -s tests/distributed/test_packed_tensor.py + - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py + - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py + - pytest -v -s tests/distributed/test_rocm_quick_reduce.py + - pytest -v -s tests/distributed/test_quick_all_reduce.py - label: ":amd: (MI300) Distributed" # TBD timeout_in_minutes: 40 @@ -993,7 +1138,7 @@ steps: #---------------------------------------------------------- mi300 · engine -----------------------------------------------------------# - label: ":amd: (MI300) Engine Core" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1008,6 +1153,17 @@ steps: - tests/test_vllm_port - tests/jit_monitor/test_hooks.py - tests/jit_monitor/test_hooks_gpu.py + - vllm/compilation/ + - vllm/config/ + - vllm/engine/ + - vllm/entrypoints/logger.py + - vllm/envs.py + - vllm/logger.py + - vllm/logging_utils/ + - vllm/platforms/ + - vllm/sequence.py + - vllm/triton_utils/ + - vllm/utils/ commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py jit_monitor/test_hooks_gpu.py @@ -1049,8 +1205,9 @@ steps: - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests -- label: ":amd: (MI300) Entrypoints Integration (API Server)" # TBD +- label: ":amd: (MI300) Entrypoints Integration (API Server) %N" timeout_in_minutes: 65 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1063,12 +1220,12 @@ steps: - tests/entrypoints/scale_out commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc - - pytest -v -s entrypoints/scale_out + - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi + - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI300) Entrypoints Integration (OpenAI API completion)" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1081,10 +1238,10 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - - pytest -v -s entrypoints/openai/ --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness - label: ":amd: (MI300) Entrypoints Integration (OpenAI API chat_completion)" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1114,7 +1271,6 @@ steps: - tests/entrypoints/cohere - tests/entrypoints/generate commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s tool_use - pytest -v -s entrypoints/tool_parsers - pytest -v -s entrypoints/generate @@ -1122,7 +1278,7 @@ steps: - pytest -v -s entrypoints/cohere - label: ":amd: (MI300) Entrypoints Integration (Responses API)" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1133,7 +1289,6 @@ steps: - vllm/ - tests/entrypoints/openai/responses commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/responses - label: ":amd: (MI300) Entrypoints Integration (Speech to Text)" # TBD @@ -1146,6 +1301,7 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/speech_to_text + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text @@ -1181,7 +1337,7 @@ steps: - pytest -v -s entrypoints/pooling - label: ":amd: (MI300) OpenAI API Correctness" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1220,11 +1376,12 @@ steps: - .buildkite/scripts/scheduled_integration_test/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030 - label: ":amd: (MI300) LM Eval Small Models" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1238,11 +1395,12 @@ steps: - vllm/v1/attention/selector.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt - label: ":amd: (MI300) MRCR Eval Small Models" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1275,11 +1433,12 @@ steps: - vllm/v1/attention/selector.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - .buildkite/lm-eval-harness/ commands: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt - label: ":amd: (MI300) Multimodal Accuracy Eval (Small Models)" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1295,7 +1454,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 - label: ":amd: (MI300) GPQA Eval (GPT-OSS)" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -1317,7 +1476,7 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt -- label: ":amd: (MI300) LM Eval Small Models FP8 + Mixed" # TBD +- label: ":amd: (MI300) LM Eval Small Models Distributed" timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -1333,6 +1492,8 @@ steps: - vllm/v1/attention/selector.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt + optional: true commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt @@ -1517,7 +1678,7 @@ steps: #--------------------------------------------------------- mi300 · examples ----------------------------------------------------------# - label: ":amd: (MI300) Examples" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1529,27 +1690,27 @@ steps: - examples/ - vllm/platforms/rocm.py commands: - - pip install tensorizer # Basic - - python3 basic/offline_inference/chat.py - - python3 basic/offline_inference/generate.py --model facebook/opt-125m - - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 - - python3 basic/offline_inference/classify.py - - python3 basic/offline_inference/embed.py - - python3 basic/offline_inference/score.py # Multi-modal models - - python3 generate/multimodal/audio_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 # Pooling models - - python3 pooling/embed/vision_embedding_offline.py --seed 0 # Features demo - - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 deployment/llm_engine_example.py - - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 + - pip install --no-deps tensorizer + - python3 basic/offline_inference/chat.py + - python3 basic/offline_inference/generate.py --model facebook/opt-125m + - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 + - python3 basic/offline_inference/classify.py + - python3 basic/offline_inference/embed.py + - python3 basic/offline_inference/score.py + - python3 generate/multimodal/audio_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 + - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 + - python3 pooling/embed/vision_embedding_offline.py --seed 0 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py + - python3 deployment/llm_engine_example.py + - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 #---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------# @@ -1565,6 +1726,8 @@ steps: - vllm/kernels - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/ir/ + - tests/kernels/ir/ commands: - pytest -v -s tests/ir - pytest -v -s tests/kernels/ir @@ -1584,15 +1747,16 @@ steps: - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py + optional: true commands: - - rocm-smi + - amd-smi - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py - label: ":amd: (MI300) Attention Kernels Shard %N" # TBD - timeout_in_minutes: 70 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1607,27 +1771,10 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + - vllm/utils/flashinfer.py commands: - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI300) Core Operation Kernels Shard %N" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - parallelism: 3 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - csrc/ - - tests/kernels/core - - tests/kernels/test_top_k_per_row.py - - tests/kernels/test_concat_mla_q.py - - vllm/model_executor/layers/rotary_embedding/ - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - - label: ":amd: (MI300) MiniMax Reduce RMS Kernels" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1643,6 +1790,12 @@ steps: - vllm/platforms/rocm.py - tests/kernels/core/test_minimax_reduce_rms.py - tests/utils.py + - csrc/libtorch_stable/minimax_reduce_rms_kernel.cu + - csrc/libtorch_stable/minimax_reduce_rms_kernel.h + - csrc/libtorch_stable/ops.h + - csrc/libtorch_stable/torch_bindings.cpp + - vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py + optional: true commands: - pytest -v -s kernels/core/test_minimax_reduce_rms.py @@ -1667,7 +1820,7 @@ steps: - pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py - label: ":amd: (MI300) Mamba Kernels" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1682,7 +1835,7 @@ steps: - pytest -v -s kernels/mamba - label: ":amd: (MI300) MoE Kernels Shard %N" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1708,8 +1861,9 @@ steps: - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 pytest -v -s kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI300) Quantization Kernels" # TBD - timeout_in_minutes: 55 +- label: ":amd: (MI300) Quantization Kernels Shard %N" + timeout_in_minutes: 120 + parallelism: 6 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1729,8 +1883,10 @@ steps: - vllm/platforms/rocm.py - vllm/model_executor/kernels/ - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/config/ + - tests/kernels/quantization/test_rocm_skinny_gemms.py commands: - - pytest -v -s kernels/quantization + - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI300) DeepEP FP8 MoE Kernels" # TBD timeout_in_minutes: 55 @@ -1754,7 +1910,7 @@ steps: #----------------------------------------------------------- mi300 · lora ------------------------------------------------------------# - label: ":amd: (MI300) LoRA Shard %N" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 85 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1768,8 +1924,9 @@ steps: commands: - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py -- label: ":amd: (MI300) LoRA TP (Distributed)" # TBD +- label: ":amd: (MI300) LoRA TP (Distributed) %N" timeout_in_minutes: 65 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -1780,14 +1937,17 @@ steps: - vllm/model_executor/layers/fused_moe/ - tests/lora - vllm/platforms/rocm.py + optional: true commands: - - pytest -v -s -x lora/test_chatglm3_tp.py - - pytest -v -s -x lora/test_llama_tp.py - - pytest -v -s -x lora/test_qwen3_with_multi_loras.py - - pytest -v -s -x lora/test_olmoe_tp.py - - pytest -v -s -x lora/test_gptoss_tp.py - - pytest -v -s -x lora/test_qwen35_densemodel_lora.py - - pytest -v -s -x lora/test_gemma4_tp.py + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi #------------------------------------------------------ mi300 · model_executor -------------------------------------------------------# @@ -1806,160 +1966,19 @@ steps: - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - vllm/model_executor/warmup + - tests/model_executor/test_jit_warmup.py + - tests/model_executor/test_jit_warmup_cutedsl_launcher.py + - tests/model_executor/test_jit_warmup_triton_launcher.py commands: - apt-get update && apt-get install -y curl libsodium23 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s model_executor -m '(not slow_test)' - - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py - -#---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------# - -- label: ":amd: (MI300) Model Runner V2 Core" # TBD - timeout_in_minutes: 55 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/worker/gpu_worker.py - - vllm/v1/core/sched/ - - vllm/v1/attention/ - - tests/v1/engine/test_llm_engine.py - - tests/v1/e2e/ - - tests/entrypoints/llm/test_struct_output_generate.py - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" - - pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" - - pytest -v -s v1/e2e/general/test_context_length.py - - pytest -v -s v1/e2e/general/test_min_tokens.py - - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" - -- label: ":amd: (MI300) Model Runner V2 Examples" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/examples" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/core/sched/ - - vllm/v1/worker/gpu_worker.py - - examples/basic/offline_inference/ - - examples/generate/multimodal/ - - examples/features/ - - examples/pooling/embed/vision_embedding_offline.py - - examples/features/tensorize_vllm_model.py - - examples/deployment/llm_engine_example.py - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - pip install tensorizer - - python3 basic/offline_inference/chat.py - - python3 basic/offline_inference/generate.py --model facebook/opt-125m - - python3 generate/multimodal/audio_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 - - python3 pooling/embed/vision_embedding_offline.py --seed 0 - - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 deployment/llm_engine_example.py - - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 - -- label: ":amd: (MI300) Model Runner V2 Distributed" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - num_gpus: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/worker/gpu_worker.py - - tests/basic_correctness/test_basic_correctness.py - - tests/v1/distributed/test_async_llm_dp.py - - tests/v1/distributed/test_eagle_dp.py - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True" - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray" - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - -- label: ":amd: (MI300) Model Runner V2 Pipeline Parallelism" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_4 - num_gpus: 4 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/worker/gpu_worker.py - - tests/distributed/test_pipeline_parallel.py - - tests/distributed/test_pp_cudagraph.py - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - pytest -v -s distributed/test_pipeline_parallel.py -k "not ray" - - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray" - -- label: ":amd: (MI300) Model Runner V2 Spec Decode" # TBD - timeout_in_minutes: 90 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/worker/gpu_worker.py - - tests/v1/spec_decode/test_max_len.py - - tests/v1/spec_decode/test_rejection_sampler_utils.py - - tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py - - tests/v1/e2e/spec_decode/ - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp" - - pytest -v -s v1/spec_decode/test_rejection_sampler_utils.py - - pytest -v -s v1/spec_decode/test_synthetic_rejection_sampler_utils.py - - pytest -v -s v1/e2e/spec_decode/eagle/ - - pytest -v -s v1/e2e/spec_decode/speculators/ - - pytest -v -s v1/e2e/spec_decode/mtp/ + - export PYTHONFAULTHANDLER=1 + - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread + - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread #------------------------------------------------------ mi300 · models / basic -------------------------------------------------------# -- label: ":amd: (MI300) Basic Models (Extra Initialization) Shard %N" # TBD - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - parallelism: 6 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/model_executor/models/ - - vllm/model_executor/layers/ - - tests/models/test_initialization.py - - tests/models/registry.py - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - - label: ":amd: (MI300) Basic Models (Initialization)" # TBD timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1970,6 +1989,7 @@ steps: - vllm/ - tests/models/test_initialization.py - tests/models/registry.py + optional: true commands: - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset @@ -1987,8 +2007,9 @@ steps: - tests/models/test_registry.py - tests/models/test_hyv4_rocm.py - tests/models/test_vision.py + - tests/models/test_deepseek_v4_vl_rocm.py commands: - - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py + - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'not distributed' - pytest -v -s models/test_vision.py::test_simple_mrope_vision_model_spatial_merge @@ -2007,8 +2028,9 @@ steps: commands: - pytest -v -s models/language/pooling_mteb_test -- label: ":amd: (MI300) Language Models (Extended Pooling)" # TBD - timeout_in_minutes: 80 +- label: ":amd: (MI300) Language Models (Extended Pooling) Shard %N" + timeout_in_minutes: 95 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2018,10 +2040,10 @@ steps: - vllm/ - tests/models/language/pooling commands: - - pytest -v -s models/language/pooling -m 'not core_model' + - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI300) Language Models (Standard)" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2072,22 +2094,6 @@ steps: - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' -- label: ":amd: (MI300) Language Models (Hybrid) Shard %N" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - parallelism: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/language/generation - commands: - - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' - - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - #---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------# - label: ":amd: (MI300) Multimodal Models (Extended Generation 1)" # TBD @@ -2105,8 +2111,9 @@ steps: - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py - pytest -v -s models/multimodal/test_mapping.py -- label: ":amd: (MI300) Multimodal Models (Extended Generation 2)" # TBD - timeout_in_minutes: 65 +- label: ":amd: (MI300) Multimodal Models (Extended Generation 2) Shard %N" + timeout_in_minutes: 100 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2116,11 +2123,11 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' + - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI300) Multimodal Models (Extended Generation 3)" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2133,7 +2140,7 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' - label: ":amd: (MI300) Multimodal Models (Standard) 1: qwen2" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2147,7 +2154,7 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model - label: ":amd: (MI300) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2157,27 +2164,13 @@ steps: - vllm/ - tests/models/multimodal/generation - tests/models/multimodal/test_mapping.py + - tests/models/multimodal commands: - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model -- label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper" # TBD - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing - - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model - - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model - - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model - - label: ":amd: (MI300) Multimodal Processor Shard %N" # TBD - timeout_in_minutes: 110 + timeout_in_minutes: 115 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2205,7 +2198,7 @@ steps: - pytest -v -s models/multimodal/pooling -m 'not core_model' - label: ":amd: (MI300) Multimodal Models (Standard) 2: qwen3 + gemma" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2233,7 +2226,9 @@ steps: - vllm/platforms/rocm.py - tests/models/quantization - vllm/model_executor/model_loader/ + optional: true commands: + - unset VLLM_USE_V2_MODEL_RUNNER - pytest -v -s models/quantization #-------------------------------------------------- mi300 · models / transformers ---------------------------------------------------# @@ -2311,7 +2306,7 @@ steps: #---------------------------------------------------------- mi300 · plugins ----------------------------------------------------------# - label: ":amd: (MI300) Plugin Integration" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -2324,43 +2319,43 @@ steps: - vllm/platforms/rocm.py commands: # BEGIN: platform plugin and general plugin tests, all the code in-between runs on dummy platform + # END: platform plugin tests + # BEGIN: `io_processor` plugins test, all the code in between uses the `prithvi_io_processor` plugin + # END: `io_processor` plugins test + # BEGIN: `bge_m3_sparse io_processor` test + # END: `bge_m3_sparse io_processor` test + # BEGIN: `colbert_query io_processor` test + # END: `colbert_query io_processor` test + # BEGIN: `stat_logger` plugins test + # END: `stat_logger` plugins test + # BEGIN: `endpoint` plugins test + # END: `endpoint` plugins test + # BEGIN: other tests - pip install -e ./plugins/vllm_add_dummy_platform - pytest -v -s plugins_tests/test_platform_plugins.py - pip uninstall vllm_add_dummy_platform -y - # END: platform plugin tests - # BEGIN: `io_processor` plugins test, all the code in between uses the `prithvi_io_processor` plugin + - pytest -v -s ./plugins_tests/test_io_processor_plugins.py - pip install -e ./plugins/prithvi_io_processor_plugin - - pytest -v -s plugins_tests/test_io_processor_plugins.py - pytest -v -s plugins_tests/test_terratorch_io_processor_plugins.py - pip uninstall prithvi_io_processor_plugin -y - # END: `io_processor` plugins test - # BEGIN: `bge_m3_sparse io_processor` test - pip install -e ./plugins/bge_m3_sparse_plugin - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py - pip uninstall bge_m3_sparse_plugin -y - # END: `bge_m3_sparse io_processor` test - # BEGIN: `colbert_query io_processor` test - pip install -e ./plugins/colbert_query_plugin - pytest -v -s plugins_tests/test_colbert_query_io_processor_plugins.py - pip uninstall colbert_query_plugin -y - # END: `colbert_query io_processor` test - # BEGIN: `stat_logger` plugins test - pip install -e ./plugins/vllm_add_dummy_stat_logger - pytest -v -s plugins_tests/test_stats_logger_plugins.py - pip uninstall dummy_stat_logger -y - # END: `stat_logger` plugins test - # BEGIN: `endpoint` plugins test - pip install -e ./plugins/vllm_add_dummy_endpoint_plugin - pytest -v -s plugins_tests/test_endpoint_plugins.py - pip uninstall vllm_add_dummy_endpoint_plugin -y - # END: `endpoint` plugins test - # BEGIN: other tests - pytest -v -s plugins_tests/test_scheduler_plugins.py - pip install -e ./plugins/vllm_add_dummy_model - pytest -v -s distributed/test_distributed_oot.py - - pytest -v -s plugins_tests/test_oot_registration_online.py # it needs a clean process - - pytest -v -s plugins_tests/test_oot_registration_offline.py # it needs a clean process - - pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins + - pytest -v -s plugins_tests/test_oot_registration_online.py + - pytest -v -s plugins_tests/test_oot_registration_offline.py + - pytest -v -s plugins_tests/lora_resolvers - label: ":amd: (MI300) GGUF Plugin" # TBD timeout_in_minutes: 45 @@ -2402,18 +2397,24 @@ steps: - tests/entrypoints/serve/instrumentator/test_uds.py - tests/v1/sample/test_logprobs_e2e.py - vllm/platforms/rocm.py + - tests/entrypoints/openai/chat_completion/test_include_reasoning.py + - tests/entrypoints/openai/chat_completion/test_serving_chat.py + - tests/entrypoints/openai/completion/test_shutdown.py + - tests/entrypoints/openai/test_uds.py commands: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)" - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming" + - pytest -v -s entrypoints/openai/chat_completion/test_include_reasoning.py - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple" + - pytest -v -s entrypoints/openai/chat_completion/test_serving_chat.py -k "test_chat_per_request_metrics_follow_server_flag or test_streaming_reasoning_usage_counts_across_deltas or test_completion_tokens_details" - pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison" - pytest -v -s entrypoints/serve/instrumentator/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" -- label: ":amd: (MI300) Rust Frontend Serve + Admin Coverage" # TBD +- label: ":amd: (MI300) Rust Frontend Serve/Admin Coverage" timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -2509,8 +2510,9 @@ steps: #------------------------------------------------------- mi300 · quantization --------------------------------------------------------# -- label: ":amd: (MI300) Quantization" # TBD +- label: ":amd: (MI300) Quantization Shard %N" timeout_in_minutes: 115 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2524,19 +2526,19 @@ steps: - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py - tests/quantization - tests/rocm/test_moe_weight_replay.py + optional: true commands: - # temporary install here since we need nightly, will move to requirements/test.in # after torchao 0.12 release, and pin a working version of torchao nightly here - # since torchao nightly is only compatible with torch nightly currently # https://github.com/pytorch/ao/issues/2919, we'll have to skip new torchao tests for now # we can only upgrade after this is resolved # TODO(jerryzh168): resolve the above comment + - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.17.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py - - pytest -v -s rocm/test_moe_weight_replay.py + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi #--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------# @@ -2556,6 +2558,7 @@ steps: - tests/conftest.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - vllm/entrypoints/generate/beam_search commands: - pytest -v -s samplers @@ -2570,6 +2573,18 @@ steps: source_file_dependencies: - vllm/ - tests/test_regression + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py @@ -2586,6 +2601,8 @@ steps: - requirements/ - setup.py - vllm/platforms/rocm.py + - .buildkite/scripts/check-ray-compatibility.sh + optional: true commands: - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh @@ -2608,7 +2625,7 @@ steps: - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test - label: ":amd: (MI300) E2E Scheduling" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2622,7 +2639,7 @@ steps: - pytest -v -s v1/e2e/general/test_async_scheduling.py - label: ":amd: (MI300) V1 Engine" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2632,12 +2649,14 @@ steps: - vllm/v1/engine/ - tests/v1/engine/ - vllm/platforms/rocm.py + - tests/v1/test_tensor_ipc_queue.py commands: - pytest -v -s v1/engine/test_preprocess_error_handling.py - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py + - pytest -v -s v1/test_tensor_ipc_queue.py - label: ":amd: (MI300) Spec Decode Draft Model" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2654,7 +2673,7 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode/draft_model/ -- label: ":amd: (MI300) Spec Decode Eagle" # TBD +- label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -2670,7 +2689,26 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py commands: - - pytest -v -s v1/e2e/spec_decode/eagle/ + - pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3" + +- label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3" + - label: ":amd: (MI300) Spec Decode N-Gram + Suffix" # TBD timeout_in_minutes: 35 @@ -2693,7 +2731,7 @@ steps: - python3 spec_decode/test_custom_proposer.py - label: ":amd: (MI300) Spec Decode Speculators + MTP" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2708,12 +2746,13 @@ steps: - vllm/transformers_utils/configs/speculators/ - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py + - vllm/v1/attention/backends/ commands: - pytest -v -s v1/e2e/spec_decode/speculators/ - pytest -v -s v1/e2e/spec_decode/mtp/ -- label: ":amd: (MI300) Speculators Correctness" # TBD - timeout_in_minutes: 35 +- label: ":amd: (MI300) Speculators Correctness Nightly" + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2740,6 +2779,7 @@ steps: - tests/evals/gsm8k/ - tests/v1/spec_decode/test_speculators_correctness.py - vllm/platforms/rocm.py + - vllm/v1/worker/gpu/spec_decode/dflash/ commands: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test @@ -2753,6 +2793,17 @@ steps: source_file_dependencies: - vllm/ - tests/v1/spec_decode + - vllm/config/ + - vllm/distributed/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/models/qwen4_exp/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not slow_test' v1/spec_decode @@ -2763,7 +2814,7 @@ steps: dind: false agent_pool: mi300_2 num_gpus: 2 - working_dir: "/vllm-workspace/" + working_dir: /vllm-workspace source_file_dependencies: - vllm/config/speculative.py - vllm/distributed/kv_transfer/kv_connector/ @@ -2785,12 +2836,15 @@ steps: - vllm/_aiter_ops.py - tests/v1/kv_connector/extract_hidden_states_integration/ - vllm/platforms/rocm.py + - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py + - tests/v1/kv_connector/extract_hidden_states_integration + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration + - pytest -v -s -m 'distributed' tests/v1/kv_connector/extract_hidden_states_integration - label: ":amd: (MI300) V1 Attention Shard %N" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 125 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2808,7 +2862,7 @@ steps: commands: - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI300) V1 Core + KV + Metrics" # TBD +- label: ":amd: (MI300) V1 Core" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -2826,23 +2880,204 @@ steps: - tests/v1/ec_connector/unit - tests/v1/metrics - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # - export HSA_NO_SCRATCH_RECLAIM=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/core + +- label: ":amd: (MI300) V1 Executor + Worker" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/executor - - pytest -v -s v1/kv_offload - - pytest -v -s v1/simple_kv_offload - pytest -v -s v1/worker - pytest -v -s v1/streaming_input - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit - - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit + optional: true + +- label: ":amd: (MI300) V1 KV Offload" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh + - pytest -v -s v1/kv_offload + - pytest -v -s v1/simple_kv_offload + optional: true + +- label: ":amd: (MI300) V1 KV Connectors Shard %N" + timeout_in_minutes: 60 + parallelism: 4 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + optional: true + +- label: ":amd: (MI300) V1 Metrics + LM Eval" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export GIT_CONFIG_COUNT=1 + - export GIT_CONFIG_KEY_0=http.version + - export GIT_CONFIG_VALUE_0=HTTP/1.1 + - export GIT_TERMINAL_PROMPT=0 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/metrics - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - # - export HSA_NO_SCRATCH_RECLAIM=1 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine + optional: true -- label: ":amd: (MI300) V1 Sample + Logits" # TBD + +- label: ":amd: (MI300) V1 Sample" timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -2856,41 +3091,56 @@ steps: - tests/v1/test_oracle.py - tests/v1/test_request.py - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/sample + +- label: ":amd: (MI300) V1 Logits + Oracle" + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/sample + - tests/v1/logits_processors + - tests/v1/test_oracle.py + - tests/v1/test_request.py + - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/logits_processors - pytest -v -s v1/test_oracle.py - pytest -v -s v1/test_request.py - pytest -v -s v1/test_outputs.py -- label: ":amd: (MI300) Distributed DP Basic" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - optional: true - num_gpus: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/ - - vllm/engine/ - - vllm/executor/ - - vllm/worker/worker_base.py - - vllm/v1/engine/ - - vllm/v1/worker/ - - tests/v1/distributed - - tests/entrypoints/launchers/api_server/test_multi_api_servers.py - - vllm/platforms/rocm.py - commands: - - export NCCL_CUMEM_HOST_ENABLE=0 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py - - DP_SIZE=2 pytest -v -s entrypoints/launchers/api_server/test_multi_api_servers.py - label: ":amd: (MI300) Metrics, Tracing" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -2901,6 +3151,18 @@ steps: - vllm/ - tests/v1/tracing - tests/tracing/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tracing/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: - "pip install \ 'opentelemetry-sdk>=1.26.0' \ @@ -2911,7 +3173,7 @@ steps: - pytest -v -s tracing - label: ":amd: (MI300) V1 E2E" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -2921,31 +3183,32 @@ steps: source_file_dependencies: - vllm/ - tests/v1/e2e + - vllm/compilation/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/envs.py + - vllm/forward_context.py + - vllm/inputs/ + - vllm/logger.py + - vllm/logging_utils/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/triton_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/e2e/spec_decode/ commands: - >- pytest -v -s v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_tensor_parallelism v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_engine_args_tensor_parallelism -- label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - num_gpus: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - - vllm/v1/worker/kv_connector_model_runner_mixin.py - - tests/v1/kv_connector/nixl_integration/ - - vllm/platforms/rocm.py - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh - - label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -2984,38 +3247,6 @@ steps: - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp - pytest -v -s distributed/test_utils.py -- label: ":amd: (MI300) Distributed NixlConnector PD accuracy" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_4 - num_gpus: 4 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - - tests/v1/kv_connector/nixl_integration/ - - vllm/platforms/rocm.py - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - -- label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" # TBD - timeout_in_minutes: 30 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_4 - num_gpus: 4 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - - tests/v1/kv_connector/nixl_integration/ - - vllm/platforms/rocm.py - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - - label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -3089,16 +3320,18 @@ steps: - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh - label: ":amd: (MI300) V1 E2E Hybrid Chunked Prefill" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 num_gpus: 4 optional: true source_file_dependencies: - - vllm/v1/attention/backends/utils.py - - vllm/v1/worker/gpu_model_runner.py - - tests/v1/e2e/test_hybrid_chunked_prefill.py + - vllm/v1/attention/backends/utils.py + - vllm/v1/worker/gpu_model_runner.py + - tests/v1/e2e/test_hybrid_chunked_prefill.py + - vllm/v1/core/ + working_dir: /vllm-workspace/tests commands: - pytest -v -s v1/e2e/test_hybrid_chunked_prefill.py @@ -3114,6 +3347,7 @@ steps: source_file_dependencies: - vllm/ - tests/weight_loading + optional: true commands: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt @@ -3156,11 +3390,90 @@ steps: #-------------------------------------------------------- mi355 · distributed --------------------------------------------------------# -- label: ":amd: (MI355) Distributed Features" # TBD +- label: ":amd: (MI355) Distributed DP Basic" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_2 + optional: true + num_gpus: 2 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/ + - vllm/engine/ + - vllm/executor/ + - vllm/worker/worker_base.py + - vllm/v1/engine/ + - vllm/v1/worker/ + - tests/v1/distributed + - tests/entrypoints/launchers/api_server/test_multi_api_servers.py + - vllm/platforms/rocm.py + - tests/v1/e2e/general/test_sharded_sampling.py + - tests/v1/e2e/spec_decode/test_sharded_sampling.py + - tests/v1/e2e/spec_decode/utils.py + commands: + - export NCCL_CUMEM_HOST_ENABLE=0 + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py + - DP_SIZE=2 pytest -v -s entrypoints/launchers/api_server/test_multi_api_servers.py + - pytest -v -s v1/e2e/general/test_sharded_sampling.py + - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py + +- label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance" # TBD + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/v1/worker/kv_connector_model_runner_mixin.py + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh + +- label: ":amd: (MI355) Distributed NixlConnector PD accuracy" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" # TBD + timeout_in_minutes: 40 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: ":amd: (MI355) Distributed Features" # TBD + timeout_in_minutes: 90 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 num_gpus: 2 optional: true working_dir: "/vllm-workspace/" @@ -3191,18 +3504,28 @@ steps: - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py - tests/v1/distributed/test_dbo.py - tests/utils.py + - examples/features/data_parallel/data_parallel_offline.py + - examples/rl/rlhf_async_new_apis.py + - tests/distributed/test_weight_transfer.py + - tests/distributed/test_packed_tensor.py commands: - pytest -v -s tests/distributed/test_context_parallel.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization + - pytest -v -s tests/v1/distributed/test_dbo.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py + - pytest -v -s tests/distributed/test_packed_tensor.py - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py - pytest -v -s tests/distributed/test_rocm_quick_reduce.py - pytest -v -s tests/distributed/test_quick_all_reduce.py - - pytest -v -s tests/v1/distributed/test_dbo.py #-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------# -- label: ":amd: (MI355) Entrypoints Integration (API Server)" # TBD - timeout_in_minutes: 55 +- label: ":amd: (MI355) Entrypoints Integration (API Server) %N" + timeout_in_minutes: 65 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3215,12 +3538,12 @@ steps: - tests/entrypoints/scale_out commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc - - pytest -v -s entrypoints/scale_out + - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi + - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI355) Entrypoints Integration (OpenAI API completion)" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3236,7 +3559,7 @@ steps: - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness - label: ":amd: (MI355) Entrypoints Integration (OpenAI API chat_completion)" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3251,7 +3574,7 @@ steps: - pytest -v -s entrypoints/openai/chat_completion - label: ":amd: (MI355) Entrypoints Integration (API Server Generate)" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3264,15 +3587,16 @@ steps: - tests/entrypoints/tool_parsers - tests/entrypoints/anthropic - tests/entrypoints/generate + - tests/entrypoints/cohere commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s tool_use - pytest -v -s entrypoints/tool_parsers - pytest -v -s entrypoints/generate - pytest -v -s entrypoints/anthropic + - pytest -v -s entrypoints/cohere - label: ":amd: (MI355) Entrypoints Integration (Speech to Text)" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3281,12 +3605,13 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/speech_to_text + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text - label: ":amd: (MI355) Entrypoints Integration (Multimodal)" - timeout_in_minutes: 40 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3295,12 +3620,13 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/multimodal + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/multimodal - label: ":amd: (MI355) Entrypoints Integration (Pooling)" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3309,6 +3635,7 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/pooling + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/pooling @@ -3361,8 +3688,8 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt -- label: ":amd: (MI355) LM Eval Small Models" # TBD - timeout_in_minutes: 45 +- label: ":amd: (MI355) LM Eval Small Models Distributed" + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_2 @@ -3378,6 +3705,7 @@ steps: - vllm/v1/attention/selector.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt @@ -3424,7 +3752,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4 - label: ":amd: (MI355) LM Eval Large Models" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_8 @@ -3449,7 +3777,7 @@ steps: #--------------------------------------------------------- mi355 · examples ----------------------------------------------------------# - label: ":amd: (MI355) Examples" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3459,23 +3787,24 @@ steps: - vllm/multimodal - examples/ - vllm/platforms/rocm.py + optional: true commands: - - pip install tensorizer # Basic + # Multi-modal models + # Pooling models + # Features demo + - pip install --no-deps tensorizer - python3 basic/offline_inference/chat.py - python3 basic/offline_inference/generate.py --model facebook/opt-125m - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 - python3 basic/offline_inference/classify.py - python3 basic/offline_inference/embed.py - python3 basic/offline_inference/score.py - # Multi-modal models - python3 generate/multimodal/audio_language_offline.py --seed 0 - python3 generate/multimodal/vision_language_offline.py --seed 0 - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 - # Pooling models - python3 pooling/embed/vision_embedding_offline.py --seed 0 - # Features demo - python3 features/automatic_prefix_caching/prefix_caching_offline.py - python3 deployment/llm_engine_example.py - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors @@ -3502,8 +3831,10 @@ steps: - vllm/model_executor/layers/rotary_embedding/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/kernels/test_fused_qk_norm_rope_gate.py + optional: true commands: - - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355) MiniMax Reduce RMS Kernels" # TBD timeout_in_minutes: 45 @@ -3520,6 +3851,12 @@ steps: - vllm/platforms/rocm.py - tests/kernels/core/test_minimax_reduce_rms.py - tests/utils.py + - csrc/libtorch_stable/minimax_reduce_rms_kernel.cu + - csrc/libtorch_stable/minimax_reduce_rms_kernel.h + - csrc/libtorch_stable/ops.h + - csrc/libtorch_stable/torch_bindings.cpp + - vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py + optional: true commands: - pytest -v -s kernels/core/test_minimax_reduce_rms.py @@ -3543,8 +3880,12 @@ steps: - vllm/v1/attention/selector.py - vllm/platforms/rocm.py - vllm/_aiter_ops.py + - examples/basic/offline_inference/chat.py + - tests/kernels/attention/test_attention_selector.py + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py + optional: true commands: - - rocm-smi + - amd-smi - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN - pytest -v -s tests/kernels/attention/test_attention_selector.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py @@ -3564,15 +3905,16 @@ steps: - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py + optional: true commands: - - rocm-smi + - amd-smi - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py - label: ":amd: (MI355) Attention Kernels Shard %N" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3586,11 +3928,13 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + - vllm/utils/flashinfer.py + optional: true commands: - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355) MoE Kernels Shard %N" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3610,6 +3954,7 @@ steps: - vllm/config - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py @@ -3646,11 +3991,15 @@ steps: - vllm/utils/torch_utils.py - vllm/platforms/ - vllm/_aiter_ops.py + - csrc/quantization/cutlass_w8a8/moe/ + - vllm/distributed/device_communicators/ + - vllm/config commands: - pytest -v -s kernels/moe/test_moe_layer.py -- label: ":amd: (MI355) Quantization Kernels" # TBD - timeout_in_minutes: 50 +- label: ":amd: (MI355) Quantization Kernels Shard %N" + timeout_in_minutes: 120 + parallelism: 6 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3669,8 +4018,11 @@ steps: - vllm/platforms/rocm.py - vllm/model_executor/kernels/ - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/config/ + - tests/kernels/quantization/test_rocm_skinny_gemms.py + optional: true commands: - - pytest -v -s kernels/quantization + - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355) FP8 MoE Kernels" # TBD timeout_in_minutes: 40 @@ -3685,6 +4037,13 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py - vllm/envs.py + - tests/kernels/moe/test_gpt_oss_triton_kernels.py + - tests/kernels/moe/test_modular_oai_triton_moe.py + - tests/kernels/moe/test_moe.py + - tests/kernels/moe/test_block_int8.py + - tests/kernels/moe/test_triton_moe_no_act_mul.py + - tests/kernels/moe/test_triton_moe_ptpc_fp8.py + optional: true commands: - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py @@ -3694,7 +4053,7 @@ steps: - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py - label: ":amd: (MI355) DeepEP FP8 MoE Kernels" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_2 @@ -3712,24 +4071,65 @@ steps: commands: - pytest -v -s kernels/moe/test_deepep_moe.py +#------------------------------------------------------ mi355 · models / basic -------------------------------------------------------# + +- label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + parallelism: 6 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/layers/ + - tests/models/test_initialization.py + - tests/models/registry.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + optional: true + commands: + - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + #----------------------------------------------------- mi355 · models / language -----------------------------------------------------# -- label: ":amd: (MI355) Language Models (Extended Generation)" # TBD +- label: ":amd: (MI355) Language Models (Hybrid) Shard %N" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 + parallelism: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/language/generation + commands: + - export GIT_CONFIG_COUNT=1 + - export GIT_CONFIG_KEY_0=http.version + - export GIT_CONFIG_VALUE_0=HTTP/1.1 + - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' + - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' + - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + +- label: ":amd: (MI355) Language Models (Extended Generation)" # TBD + timeout_in_minutes: 95 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ - tests/models/language/generation + optional: true commands: - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@rocm-7.0-v2.3.0' - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' -- label: ":amd: (MI355) Language Models (Extended Pooling)" # TBD - timeout_in_minutes: 65 +- label: ":amd: (MI355) Language Models (Extended Pooling) Shard %N" + timeout_in_minutes: 95 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3739,10 +4139,10 @@ steps: - vllm/ - tests/models/language/pooling commands: - - pytest -v -s models/language/pooling -m 'not core_model' + - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI355) Language Models (PPL)" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3767,6 +4167,8 @@ steps: - vllm/v1/attention/ops/ - vllm/platforms/rocm.py - tests/models/language/generation_ppl_test + - vllm/model_executor/layers/fla/ops/ + - vllm/ commands: - pytest -v -s models/language/generation_ppl_test @@ -3779,6 +4181,7 @@ steps: source_file_dependencies: - vllm/ - tests/models/language + optional: true commands: - pip freeze | grep -E 'torch' - pytest -v -s models/language -m 'core_model and (not slow_test)' @@ -3786,7 +4189,7 @@ steps: #---------------------------------------------------- mi355 · models / multimodal ----------------------------------------------------# - label: ":amd: (MI355) Multimodal Models (Extended Generation 1)" # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3801,7 +4204,7 @@ steps: - pytest -v -s models/multimodal/test_mapping.py - label: ":amd: (MI355) Multimodal Models (Extended Generation 3)" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3814,7 +4217,7 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' - label: ":amd: (MI355) Multimodal Models (Extended Pooling)" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3827,7 +4230,7 @@ steps: - pytest -v -s models/multimodal/pooling -m 'not core_model' - label: ":amd: (MI355) Multimodal Models (Standard) 1: qwen2" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3841,7 +4244,7 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model - label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3853,13 +4256,13 @@ steps: commands: - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model - - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model + - pytest models/multimodal/generation/test_memory_leak.py -m core_model - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model #----------------------------------------------------- mi355 · models / quantized -----------------------------------------------------# - label: ":amd: (MI355) Quantized Models" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3870,13 +4273,16 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py - vllm/model_executor/model_loader/ + optional: true commands: + - unset VLLM_USE_V2_MODEL_RUNNER - pytest -v -s models/quantization #------------------------------------------------------- mi355 · quantization --------------------------------------------------------# -- label: ":amd: (MI355) Quantization" # TBD - timeout_in_minutes: 80 +- label: ":amd: (MI355) Quantization Shard %N" + timeout_in_minutes: 115 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3887,10 +4293,16 @@ steps: - tests/quantization - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - vllm/model_executor/layers/fused_moe/oracle/unquantized.py + - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py + - tests/rocm/test_moe_weight_replay.py + optional: true commands: + - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.17.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi # - label: Quantized MoE Test (B200-MI355) # TBD # timeout_in_minutes: 180 @@ -3922,7 +4334,7 @@ steps: #------------------------------------------------------------ mi355 · v1 -------------------------------------------------------------# - label: ":amd: (MI355) V1 Attention Shard %N" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 125 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3936,11 +4348,12 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + optional: true commands: - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI355) V1 Core + KV + Metrics" # TBD - timeout_in_minutes: 55 +- label: ":amd: (MI355) V1 Core" + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3956,24 +4369,243 @@ steps: - tests/v1/kv_connector/unit - tests/v1/metrics - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/core + +- label: ":amd: (MI355) V1 Executor + Worker" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/executor - - pytest -v -s v1/kv_offload - pytest -v -s v1/worker - - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit + - pytest -v -s v1/streaming_input + +- label: ":amd: (MI355) V1 KV Offload" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh + - pytest -v -s v1/kv_offload + - pytest -v -s v1/simple_kv_offload + +- label: ":amd: (MI355) V1 KV Connectors Shard %N" + timeout_in_minutes: 60 + parallelism: 4 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit + commands: + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + +- label: ":amd: (MI355) V1 Metrics + LM Eval" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit + commands: + - export GIT_CONFIG_COUNT=1 + - export GIT_CONFIG_KEY_0=http.version + - export GIT_CONFIG_VALUE_0=HTTP/1.1 + - export GIT_TERMINAL_PROMPT=0 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/metrics - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine -- label: ":amd: (MI355) V1 Sample + Logits" # TBD +- label: ":amd: (MI355) CUDAGraph" timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/v1/cudagraph_dispatcher.py + - vllm/config/compilation.py + - vllm/compilation + - vllm/platforms/rocm.py + - vllm/v1/worker/encoder_cudagraph.py + - vllm/v1/worker/encoder_cudagraph_defs.py + commands: + - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py + - pytest -v -s v1/cudagraph/test_cudagraph_mode.py + - pytest -v -s v1/cudagraph/test_breakable_cudagraph.py + - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py + + +- label: ":amd: (MI355) V1 Sample" + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -3982,15 +4614,56 @@ steps: - tests/v1/test_oracle.py - tests/v1/test_request.py - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/sample + +- label: ":amd: (MI355) V1 Logits + Oracle" + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/sample + - tests/v1/logits_processors + - tests/v1/test_oracle.py + - tests/v1/test_request.py + - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/logits_processors - pytest -v -s v1/test_oracle.py - pytest -v -s v1/test_request.py - pytest -v -s v1/test_outputs.py + - label: ":amd: (MI355) V1 Spec Decode" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3998,11 +4671,24 @@ steps: source_file_dependencies: - vllm/ - tests/v1/spec_decode + - vllm/config/ + - vllm/distributed/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/models/qwen4_exp/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not slow_test' v1/spec_decode -- label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance" # TBD - timeout_in_minutes: 180 +- label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly" + timeout_in_minutes: 55 + parallelism: 3 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -4026,13 +4712,17 @@ steps: - tests/utils.py - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py + - vllm/v1/worker/gpu/spec_decode/ + - tests/v1/e2e/spec_decode/conftest.py commands: - - pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi #------------------------------------------------------ mi355 · weight_loading -------------------------------------------------------# - label: ":amd: (MI355) Weight Loading Multi-GPU" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_2 @@ -4041,6 +4731,7 @@ steps: source_file_dependencies: - vllm/ - tests/weight_loading + optional: true commands: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt @@ -4070,6 +4761,17 @@ steps: source_file_dependencies: - vllm/ - tests/test_regression + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index 8eea800686..219a4ca018 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -92,7 +92,7 @@ steps: dind: false device: mi355_1 working_dir: "/vllm-workspace/" - timeout_in_minutes: 25 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: @@ -133,7 +133,7 @@ steps: dind: false device: mi355_1 working_dir: "/vllm-workspace/" - timeout_in_minutes: 30 + timeout_in_minutes: 40 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/cuda.yaml b/.buildkite/test_areas/cuda.yaml index 230781cd5d..a031c37ba3 100644 --- a/.buildkite/test_areas/cuda.yaml +++ b/.buildkite/test_areas/cuda.yaml @@ -23,7 +23,7 @@ steps: dind: false device: mi355_1 working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 20 + timeout_in_minutes: 25 depends_on: - image-build-amd source_file_dependencies: @@ -50,7 +50,7 @@ steps: amd: label: ":amd: (MI250) CUDAGraph" device: mi250_1 - timeout_in_minutes: 45 + timeout_in_minutes: 55 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 226d7ac94a..0fbb32f8f9 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -16,9 +16,9 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: - label: ":amd: (MI300) Distributed NixlConnector PD accuracy" + label: ":amd: (MI355) Distributed NixlConnector PD accuracy" dind: false - device: mi300_4 + device: mi355_4 timeout_in_minutes: 60 depends_on: - image-build-amd @@ -71,9 +71,9 @@ steps: - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: - label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" + label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" dind: false - device: mi300_4 + device: mi355_4 timeout_in_minutes: 40 depends_on: - image-build-amd @@ -130,7 +130,7 @@ steps: label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" dind: false device: mi300_4 - timeout_in_minutes: 55 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -221,9 +221,9 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh mirror: amd: - label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance" + label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance" dind: false - device: mi300_2 + device: mi355_2 timeout_in_minutes: 45 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 3ea26ce89c..b83d2893bd 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -65,9 +65,9 @@ steps: - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py mirror: amd: - label: ":amd: (MI300) Distributed DP Basic" + label: ":amd: (MI355) Distributed DP Basic" dind: false - device: mi300_2 + device: mi355_2 timeout_in_minutes: 45 depends_on: - image-build-amd @@ -125,6 +125,9 @@ steps: - tests/compile/test_wrapper.py - tests/entrypoints/llm/test_collective_rpc.py - vllm/platforms/rocm.py + commands: + - pytest -v -s entrypoints/llm/test_collective_rpc.py + - pytest -v -s ./compile/test_wrapper.py - label: ":nvidia: (L4) Distributed Torchrun + Shutdown" key: distributed-torchrun-shutdown-tests-2-gpus @@ -296,6 +299,57 @@ steps: - pytest -v -s tests/v1/distributed/test_dbo.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py - pytest -v -s tests/distributed/test_packed_tensor.py + mirror: + amd: + label: ':amd: (MI355) Distributed Features' + dind: false + device: mi355_2 + timeout_in_minutes: 90 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/ + - vllm/v1/distributed/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - tests/v1/distributed/test_dbo.py + - tests/distributed/test_context_parallel.py + - examples/features/data_parallel/data_parallel_offline.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - csrc/custom_quickreduce.cu + - csrc/ops.h + - csrc/torch_bindings.cpp + - vllm/model_executor/layers/ + - vllm/entrypoints/llm.py + - vllm/config/parallel.py + - vllm/v1/engine/ + - vllm/v1/executor/ + - vllm/v1/worker/ + - vllm/_custom_ops.py + - vllm/envs.py + - tests/distributed/test_rocm_aiter_custom_ar.py + - tests/distributed/test_rocm_quick_reduce.py + - tests/distributed/test_quick_all_reduce.py + - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py + - tests/utils.py + - examples/rl/rlhf_async_new_apis.py + - tests/distributed/test_weight_transfer.py + - tests/distributed/test_packed_tensor.py + commands: + - pytest -v -s tests/distributed/test_context_parallel.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization + - pytest -v -s tests/v1/distributed/test_dbo.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py + - pytest -v -s tests/distributed/test_packed_tensor.py + - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py + - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py + - pytest -v -s tests/distributed/test_rocm_quick_reduce.py + - pytest -v -s tests/distributed/test_quick_all_reduce.py - label: ":nvidia: (B200) Distributed" key: distributed-tests-2xb200 diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml index 02ba409e34..20bec50b5e 100644 --- a/.buildkite/test_areas/e2e_integration.yaml +++ b/.buildkite/test_areas/e2e_integration.yaml @@ -57,7 +57,7 @@ steps: dind: false device: mi355_1 working_dir: "/vllm-workspace" - timeout_in_minutes: 20 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index bb848cd9e5..179954740a 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -56,6 +56,8 @@ steps: timeout_in_minutes: 45 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) E2E Scheduling" key: e2e-scheduling-1-gpu @@ -73,6 +75,8 @@ steps: timeout_in_minutes: 55 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) E2E Core" device: h200_18gb diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 00091707d5..60ac5d932c 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -21,9 +21,11 @@ steps: label: ":amd: (MI355) Entrypoints Unit" dind: false device: mi355_1 - timeout_in_minutes: 35 + timeout_in_minutes: 50 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (LLM)" device: h200_35gb @@ -44,7 +46,7 @@ steps: label: ":amd: (MI300) Entrypoints Integration (LLM)" dind: false device: mi300_1 - timeout_in_minutes: 55 + timeout_in_minutes: 90 depends_on: - image-build-amd @@ -176,6 +178,15 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text + mirror: + amd: + label: ":amd: (MI355) Entrypoints Integration (Speech to Text)" + dind: false + device: mi355_1 + timeout_in_minutes: 60 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (Multimodal)" device: h200_35gb @@ -210,6 +221,15 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/pooling + mirror: + amd: + label: ":amd: (MI355) Entrypoints Integration (Pooling)" + dind: false + device: mi355_1 + timeout_in_minutes: 65 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd - label: ":nvidia: (H200 MIG 18GB) OpenAI API Correctness" key: openai-api-correctness diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index ed4ea67f93..5a7832d945 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -13,6 +13,22 @@ steps: commands: - pytest -v -s tests/ir - pytest -v -s tests/kernels/ir + mirror: + amd: + label: ":amd: (MI355) vLLM IR" + dind: false + device: mi355_1 + timeout_in_minutes: 50 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/ir/ + - tests/kernels/ir/ + - vllm/ir + - vllm/kernels + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Core Operation Kernels Shard %N" device: h200_35gb @@ -26,6 +42,29 @@ steps: commands: - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT parallelism: 3 + mirror: + amd: + label: ":amd: (MI355) Core Operation Kernels Shard %N" + dind: false + device: mi355_1 + timeout_in_minutes: 40 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - tests/kernels/core + - tests/kernels/test_top_k_per_row.py + - tests/kernels/test_concat_mla_q.py + - tests/kernels/test_rocm_fp32_router_gemm.py + - vllm/model_executor/layers/fused_moe/router/gate_linear.py + - vllm/model_executor/layers/fused_moe/router/rocm_fp32_router_gemm.py + - vllm/model_executor/layers/rotary_embedding/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/kernels/test_fused_qk_norm_rope_gate.py + commands: + - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":nvidia: (H100) MiniMax Reduce RMS Kernels" key: kernels-minimax-reduce-rms-test-2-gpus @@ -145,6 +184,7 @@ steps: dind: false device: mi300_1 timeout_in_minutes: 90 + parallelism: 6 depends_on: - image-build-amd source_file_dependencies: @@ -208,9 +248,6 @@ steps: label: ":amd: (MI300) Quantization Kernels Shard %N" dind: false device: mi300_1 - # Pin the AMD mirror at its current 2 shards so raising NVIDIA - # parallelism does not multiply AMD copies (needs ci-infra #473). - parallelism: 2 timeout_in_minutes: 120 source_file_dependencies: - csrc/quantization/ @@ -222,6 +259,12 @@ steps: - vllm/kernels/aiter_ops.py - vllm/platforms/rocm.py - vllm/model_executor/kernels/ + - csrc/rocm/ + - vllm/v1/attention/backends/rocm_aiter_fa.py + - tests/kernels/quant_utils.py + - tests/kernels/utils.py + - vllm/_custom_ops.py + - vllm/envs.py depends_on: - image-build-amd @@ -258,6 +301,10 @@ steps: - vllm/config - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - csrc/rocm/ + - tests/kernels/utils.py + - vllm/model_executor/layers/quantization/mxfp4.py + - vllm/model_executor/layers/quantization/utils/quant_utils.py depends_on: - image-build-amd @@ -278,6 +325,8 @@ steps: timeout_in_minutes: 55 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py - label: ":nvidia: (H100) DeepGEMM Kernels" key: kernels-deepgemm-test-h100 @@ -463,6 +512,9 @@ steps: timeout_in_minutes: 45 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py + - scripts/benchmark_helion_kernels.py - label: ":nvidia: (H100) FP8 MoE Kernels" @@ -491,6 +543,25 @@ steps: commands: - pytest -v -s kernels/moe/test_deepep_deepgemm_moe.py - pytest -v -s kernels/moe/test_deepep_moe.py + mirror: + amd: + label: ":amd: (MI355) DeepEP FP8 MoE Kernels" + dind: false + device: mi355_2 + timeout_in_minutes: 55 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - csrc/quantization/w8a8/cutlass/moe/ + - vllm/model_executor/layers/fused_moe/ + - tests/kernels/moe/test_deepep_moe.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - pytest -v -s kernels/moe/test_deepep_moe.py - label: ":nvidia: (B200) FP4 MoE Kernels" key: kernels-fp4-moe-test-b200 @@ -520,7 +591,30 @@ steps: - vllm/config commands: - pytest -v -s kernels/moe/test_moe_layer.py - + mirror: + amd: + label: ":amd: (MI355) FusedMoE Layer Kernels" + dind: false + device: mi355_2 + timeout_in_minutes: 35 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - csrc/rocm/ + - tests/kernels/moe + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/ + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/utils/import_utils.py + - vllm/utils/math_utils.py + - vllm/utils/torch_utils.py + - vllm/platforms/ + - vllm/_aiter_ops.py - label: ":nvidia: (B200) FusedMoE Layer Kernels" key: kernels-fusedmoe-layer-test-2-b200s @@ -538,3 +632,90 @@ steps: commands: - pytest -v -s kernels/moe/test_moe_layer.py - pytest -v -s kernels/moe/test_deepep_v2_moe.py + +- label: ":amd: (MI355) MLA Kernels" + key: amd-mla-kernels-mi355 + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 25 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/attention/backends/mla/rocm_aiter_mla.py + - vllm/v1/attention/selector.py + - vllm/platforms/rocm.py + - vllm/_aiter_ops.py + - tests/kernels/attention/test_rocm_aiter_mla_decode.py + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py + - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py + - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py + commands: + - amd-smi + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py + +- label: ":amd: (MI355) FP8 MoE Kernels" + key: amd-fp8-moe-kernels-mi355 + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 40 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - vllm/model_executor/layers/fused_moe/ + - tests/kernels/moe/test_deepep_moe.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + - tests/kernels/moe/test_gpt_oss_triton_kernels.py + - tests/kernels/moe/test_modular_oai_triton_moe.py + - tests/kernels/moe/test_moe.py + - tests/kernels/moe/test_block_int8.py + - tests/kernels/moe/test_triton_moe_no_act_mul.py + - tests/kernels/moe/test_triton_moe_ptpc_fp8.py + commands: + - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py + - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py + - pytest -v -s kernels/moe/test_moe.py + - pytest -v -s kernels/moe/test_block_int8.py + - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py + - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py + +- label: ":amd: (MI355) Kernels" + key: amd-kernels-mi355 + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 25 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/quantization/fp4/ + - csrc/attention/mla/ + - csrc/quantization/cutlass_w8a8/moe/ + - vllm/model_executor/layers/fused_moe/cutlass_moe.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/backends/rocm_attn.py + - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/v1/attention/backends/mla/aiter_triton_mla.py + - vllm/v1/attention/backends/mla/rocm_aiter_mla.py + - vllm/v1/attention/selector.py + - vllm/platforms/rocm.py + - vllm/_aiter_ops.py + - examples/basic/offline_inference/chat.py + - tests/kernels/attention/test_attention_selector.py + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py + commands: + - amd-smi + - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN + - pytest -v -s tests/kernels/attention/test_attention_selector.py + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index ec45f7e69e..e94cbe5dfc 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -30,6 +30,27 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py +- label: ":amd: (MI300) LM Eval Small Models Harness" + key: amd-lm-eval-small-models-harness + dind: false + device: mi300_1 + timeout_in_minutes: 35 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/.buildkite/lm-eval-harness + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - .buildkite/lm-eval-harness/ + commands: + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt + - label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking" device: h200_35gb key: lm-eval-watermarking @@ -100,7 +121,7 @@ steps: label: ":amd: (MI355) LM Eval Small Models Distributed" dind: false device: mi355_2 - timeout_in_minutes: 45 + timeout_in_minutes: 55 depends_on: - image-build-amd source_file_dependencies: @@ -286,13 +307,16 @@ steps: label: ":amd: (MI300) LM Eval Large Models" dind: false device: mi300_8 - timeout_in_minutes: 40 + timeout_in_minutes: 65 depends_on: - image-build-amd commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt + source_file_dependencies: + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) MoE Refactor Integration TEMPORARY" key: moe-refactor-integration-test-h100-temporary diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 93fb5f5b81..28f695f391 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -61,7 +61,7 @@ steps: dind: false device: mi300_4 working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 25 + timeout_in_minutes: 65 depends_on: - image-build-amd source_file_dependencies: @@ -73,11 +73,11 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn # expandable_segments breaks custom all-reduce IPC on ROCm. - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac - - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_chatglm3_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_llama_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_qwen3_with_multi_loras.py - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_olmoe_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp; fi diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 940447bf1b..f17d154fbe 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -99,7 +99,7 @@ steps: label: ":amd: (MI300) V1 Core" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -119,7 +119,7 @@ steps: label: ":amd: (MI300) V1 Executor + Worker" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -138,7 +138,7 @@ steps: label: ":amd: (MI300) V1 KV Offload" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -158,7 +158,7 @@ steps: label: ":amd: (MI300) V1 KV Connectors Shard %N" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -258,7 +258,7 @@ steps: label: ":amd: (MI355) Extract Hidden States Integration" dind: false device: mi355_2 - timeout_in_minutes: 20 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: @@ -305,6 +305,15 @@ steps: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py working_dir: "/vllm-workspace/tests" # optional + mirror: + amd: + label: ":amd: (MI355) Regression" + dind: false + device: mi355_1 + timeout_in_minutes: 25 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests - label: ":nvidia: (H200 MIG 35GB) Examples" device: h200_35gb @@ -446,9 +455,15 @@ steps: label: ":amd: (MI355) Async Engine, Inputs, Utils, Worker" dind: false device: mi355_1 - timeout_in_minutes: 40 + timeout_in_minutes: 50 depends_on: - image-build-amd + commands: + - pytest -v -s detokenizer + - pytest -v -s -m 'not cpu_test' multimodal + # test_cache.py marks every test cpu_test, including this GPU regression. + - pytest -v -s multimodal/test_cache.py::test_sleep_wake_preserves_mm_cache_consistency + - pytest -v -s utils_ - label: ":computer: (CPU) Params, Env, Tokenizers, Parser" key: cpu-params-env-tokenizers-parser @@ -572,7 +587,7 @@ steps: amd: label: ":amd: (MI250) Batch Invariance" device: mi250_1 - timeout_in_minutes: 30 + timeout_in_minutes: 35 depends_on: - image-build-amd source_file_dependencies: @@ -623,3 +638,17 @@ steps: commands: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test + mirror: + amd: + label: ":amd: (MI355) Acceptance Length (Large Models)" + dind: false + device: mi355_1 + timeout_in_minutes: 45 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/model_executor/models/mlp_speculator.py + - tests/v1/spec_decode/test_acceptance_length.py + - vllm/platforms/rocm.py diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 870cc9b393..94b0723f8a 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -19,7 +19,7 @@ steps: label: ":amd: (MI355) Basic Models (Initialization)" dind: false device: mi355_1 - timeout_in_minutes: 25 + timeout_in_minutes: 30 depends_on: - image-build-amd @@ -37,6 +37,23 @@ steps: # test.) Also run if model initialization test file is modified - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB parallelism: 14 + mirror: + amd: + label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N" + dind: false + device: mi355_1 + timeout_in_minutes: 60 + parallelism: 10 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/layers/ + - tests/models/test_initialization.py + - tests/models/registry.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Basic Models (Other)" device: h200_35gb @@ -162,6 +179,6 @@ steps: device: mi250_1 no_gpu: true working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 25 + timeout_in_minutes: 35 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index f735a9b018..a054c00db8 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -38,7 +38,7 @@ steps: label: ":amd: (MI300) Distributed Models Shard %N" dind: false device: mi300_2 - timeout_in_minutes: 35 + timeout_in_minutes: 80 depends_on: - image-build-amd source_file_dependencies: @@ -63,7 +63,7 @@ steps: - | case "$$BUILDKITE_PARALLEL_JOB" in 0) - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' && HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' ;; 1) @@ -72,8 +72,8 @@ steps: VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' + pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py && + pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 68f1cdd429..3e5024fe87 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -19,7 +19,7 @@ steps: label: ":amd: (MI300) Language Models (Standard)" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -82,10 +82,10 @@ steps: parallelism: 2 mirror: amd: - label: ":amd: (MI300) Language Models (Hybrid) Shard %N" + label: ":amd: (MI355) Language Models (Hybrid) Shard %N" dind: false - device: mi300_1 - timeout_in_minutes: 60 + device: mi355_1 + timeout_in_minutes: 65 depends_on: - image-build-amd commands: @@ -133,7 +133,7 @@ steps: label: ":amd: (MI355) Language Models (Extended Generation)" dind: false device: mi355_1 - timeout_in_minutes: 70 + timeout_in_minutes: 95 depends_on: - image-build-amd commands: @@ -197,11 +197,8 @@ steps: dind: false device: mi300_1 timeout_in_minutes: 95 - parallelism: 2 depends_on: - image-build-amd - commands: - - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":nvidia: (H200 MIG 18GB) Language Models (MTEB)" key: language-models-test-mteb diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index b09e8cc2d9..e8c5ddd2cb 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -77,9 +77,9 @@ steps: - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work mirror: amd: - label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper" + label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper" dind: false - device: mi300_1 + device: mi355_1 timeout_in_minutes: 70 depends_on: - image-build-amd @@ -104,7 +104,8 @@ steps: device: mi250_1 no_gpu: true working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 80 + timeout_in_minutes: 90 + parallelism: 6 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/plugins.yaml b/.buildkite/test_areas/plugins.yaml index 65e7a0b258..11f63ff52d 100644 --- a/.buildkite/test_areas/plugins.yaml +++ b/.buildkite/test_areas/plugins.yaml @@ -75,6 +75,20 @@ steps: commands: - pip install "vllm-gguf-plugin >= 0.0.2" - pytest -v -s plugins_tests/gguf + mirror: + amd: + label: ":amd: (MI355) GGUF Plugin" + dind: false + device: mi355_1 + timeout_in_minutes: 45 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/layers/quantization + - tests/plugins_tests/test_gguf_plugin.py + - tests/plugins_tests/gguf + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) BitsAndBytes Plugin" key: bitsandbytes-plugin diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index f34ff50e67..459f7faeb4 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -122,6 +122,21 @@ steps: - tests/compile/passes commands: - pytest -s -v compile/passes --ignore compile/passes/distributed + mirror: + amd: + label: ':amd: (MI300) PyTorch Compilation Passes' + dind: false + device: mi300_1 + timeout_in_minutes: 80 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/ + - tests/compile/passes + - vllm/platforms/rocm.py + commands: + - pytest -s -v compile/passes --ignore compile/passes/distributed - label: ":nvidia: (H200 MIG 35GB) PyTorch Fullgraph" device: h200_35gb @@ -154,6 +169,7 @@ steps: - vllm/utils/ - vllm/v1/ - tests/compile + - csrc/ commands: # Run smoke tests under fullgraph directory, except test_full_cudagraph.py # as it is a heavy test that is covered in other steps. diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index 1d895e1f2f..702bbe7024 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -20,6 +20,31 @@ steps: # The SM90-only checkpoint currently contains a removed weight_chan_scale # parameter. It was not exercised by the previous L4 job. - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + mirror: + amd: + label: ":amd: (MI355) Quantization Shard %N" + dind: false + device: mi355_1 + timeout_in_minutes: 115 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - tests/quantization + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/fused_moe/oracle/unquantized.py + - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py + - tests/rocm/test_moe_weight_replay.py + commands: + # Use the ROCm default instead of the NVIDIA runner override. + - unset VLLM_USE_V2_MODEL_RUNNER + - uv pip install --system torchao==0.17.0 + - uv pip install --system conch-triton-kernels + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi - label: ":nvidia: (H200 MIG 35GB) Quantized Fusions" device: h200_35gb @@ -63,3 +88,22 @@ steps: - tests/models/quantization commands: - pytest -v -s models/quantization + mirror: + amd: + label: ":amd: (MI355) Quantized Models" + dind: false + device: mi355_1 + timeout_in_minutes: 80 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/layers/quantization + - tests/models/quantization + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/model_loader/ + commands: + # Use the ROCm default instead of the NVIDIA runner override. + - unset VLLM_USE_V2_MODEL_RUNNER + - pytest -v -s models/quantization diff --git a/.buildkite/test_areas/ray_compat.yaml b/.buildkite/test_areas/ray_compat.yaml index d24c38430b..dc8571b260 100644 --- a/.buildkite/test_areas/ray_compat.yaml +++ b/.buildkite/test_areas/ray_compat.yaml @@ -16,3 +16,19 @@ steps: - setup.py commands: - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh + mirror: + amd: + label: ":amd: (MI355) Ray Dependency Compatibility Check" + dind: false + device: mi355_1 + timeout_in_minutes: 25 + working_dir: "/" + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/scripts/check-ray-compatibility.sh + - requirements/ + - setup.py + - vllm/platforms/rocm.py + commands: + - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 770d6b64df..a679b405ed 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -136,7 +136,7 @@ steps: label: ":amd: (MI355) Rust Frontend Core Correctness" dind: false device: mi355_1 - timeout_in_minutes: 20 + timeout_in_minutes: 25 depends_on: - image-build-amd source_file_dependencies: @@ -166,7 +166,7 @@ steps: label: ":amd: (MI355) Rust Frontend Tool Use" dind: false device: mi355_1 - timeout_in_minutes: 25 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 8387270359..c06642ea1a 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -47,7 +47,7 @@ steps: label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen" dind: false device: mi300_1 - timeout_in_minutes: 35 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -74,7 +74,7 @@ steps: label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" dind: false device: mi300_1 - timeout_in_minutes: 25 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -187,7 +187,7 @@ steps: label: ":amd: (MI300) Spec Decode Draft Model" dind: false device: mi300_1 - timeout_in_minutes: 55 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -233,6 +233,9 @@ steps: timeout_in_minutes: 40 depends_on: - image-build-amd + source_file_dependencies: + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (B200) Spec Decode DeepSeek MTP Parallel Load" key: spec-decode-deepseek-mtp-parallel-load-2xb200-2xmi300 @@ -362,6 +365,33 @@ steps: # Each Gemma4 parametrization is an indivisible ~23m floor. Keep them in # separate shards and put the remaining tests in shard 0, where new tests # land by default. - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]" - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]" + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi + mirror: + amd: + label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly" + dind: false + device: mi355_1 + timeout_in_minutes: 55 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/sample/ + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_model_runner.py + - vllm/v1/worker/gpu_worker.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/models/gemma4.py + - vllm/model_executor/models/gemma4_mm.py + - vllm/model_executor/models/gemma4_mtp.py + - vllm/model_executor/models/llama.py + - vllm/model_executor/models/llama_eagle3.py + - vllm/model_executor/models/medusa.py + - vllm/model_executor/models/registry.py + - tests/evals/gsm8k/ + - tests/utils.py + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml index 154b355778..b8afa261e5 100644 --- a/.buildkite/test_areas/weight_loading.yaml +++ b/.buildkite/test_areas/weight_loading.yaml @@ -20,7 +20,7 @@ steps: label: ":amd: (MI355) Weight Loading Multi-GPU" dind: false device: mi355_2 - timeout_in_minutes: 40 + timeout_in_minutes: 70 depends_on: - image-build-amd commands: From 08633cb5cd77ea315351ae531945a9a253016788 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Thu, 17 Sep 2026 16:56:28 +0800 Subject: [PATCH 012/143] [Qwen3.8-Flash-Next] Enable FP8 TP with FlashInfer TRTLLM MoE (#55867) Signed-off-by: Canlin Signed-off-by: Isotr0py Co-authored-by: OpenAI Codex Co-authored-by: Isotr0py --- tests/kernels/moe/test_fp8_tp_loading.py | 311 ++++++++++++++++++ .../model_executor/layers/fused_moe/config.py | 2 + .../layers/fused_moe/oracle/fp8.py | 77 +++++ .../layers/fused_moe/routed_experts.py | 35 +- .../model_executor/layers/quantization/fp8.py | 43 +-- 5 files changed, 434 insertions(+), 34 deletions(-) create mode 100644 tests/kernels/moe/test_fp8_tp_loading.py diff --git a/tests/kernels/moe/test_fp8_tp_loading.py b/tests/kernels/moe/test_fp8_tp_loading.py new file mode 100644 index 0000000000..680ccb812c --- /dev/null +++ b/tests/kernels/moe/test_fp8_tp_loading.py @@ -0,0 +1,311 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest +import torch + +import vllm.distributed as distributed +from tests.kernels.moe.utils import make_dummy_moe_config +from vllm.model_executor.layers.fused_moe import ( + unquantized_fused_moe_method as unquantized_module, +) +from vllm.model_executor.layers.fused_moe.config import RoutingMethodType +from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts +from vllm.model_executor.layers.quantization import fp8 as fp8_module +from vllm.model_executor.layers.quantization.fp8 import Fp8Config + + +def _make_fp8_tp_experts( + monkeypatch, + tp_size, + tp_rank, + backend="flashinfer_trtllm", + *, + mock_backend=True, + num_experts=2, + hidden_dim=256, + intermediate_size=640, + quant_config=None, + config_overrides=None, +): + # Exercise allocation and the public weight loader without selecting a GPU kernel. + if mock_backend: + monkeypatch.setattr( + fp8_module, "select_fp8_moe_backend", lambda **kwargs: (None, None) + ) + monkeypatch.setattr( + unquantized_module, + "select_unquantized_moe_backend", + lambda **kwargs: (None, None), + ) + monkeypatch.setattr( + fp8_module, "get_tensor_model_parallel_world_size", lambda: tp_size + ) + # The block-shape validator imports this function from vllm.distributed. + monkeypatch.setattr( + distributed, "get_tensor_model_parallel_world_size", lambda: tp_size + ) + config = make_dummy_moe_config( + num_experts=num_experts, + hidden_dim=hidden_dim, + intermediate_size=intermediate_size, + experts_per_token=min(num_experts, 10), + ) + config.routing_method = RoutingMethodType.RenormalizeNaive + config.moe_parallel_config.tp_size = tp_size + config.moe_parallel_config.tp_rank = tp_rank + config.intermediate_size_per_partition = intermediate_size // tp_size + config.intermediate_size_per_partition_unpadded = intermediate_size // tp_size + config.moe_backend = backend + for field, value in (config_overrides or {}).items(): + setattr(config, field, value) + return RoutedExperts( + "model.layers.0.mlp.experts", + torch.bfloat16, + config, + quant_config + or Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]), + expert_map_manager=SimpleNamespace( + local_num_experts=num_experts, + placement_strategy="linear", + expert_map=None, + expert_mask=None, + routing_tables=None, + map_global_to_local=lambda index: index, + ), + ) + + +@pytest.mark.parametrize("tp_size", [2, 4, 8]) +@pytest.mark.parametrize("batched", [False, True]) +@pytest.mark.parametrize("intermediate_size", [640, 896]) +def test_fp8_block_aligned_tp_preserves_checkpoint( + monkeypatch, tp_size, batched, intermediate_size +): + """All ranks reconstruct the original dequantized projections, including + padding-only ranks. Reloading must clear stale weights and scales. + """ + generator = torch.Generator().manual_seed(42) + weights = { + "w1": torch.randn(2, intermediate_size, 256, generator=generator).to( + torch.float8_e4m3fn + ), + "w3": torch.randn(2, intermediate_size, 256, generator=generator).to( + torch.float8_e4m3fn + ), + "w2": torch.randn(2, 256, intermediate_size, generator=generator).to( + torch.float8_e4m3fn + ), + } + scales = { + name: torch.rand(2, w.shape[1] // 128, w.shape[2] // 128, generator=generator) + + 0.1 + for name, w in weights.items() + } + reconstructed: dict[str, list[torch.Tensor]] = {name: [] for name in weights} + num_blocks = intermediate_size // 128 + for rank in range(tp_size): + layer = _make_fp8_tp_experts( + monkeypatch, tp_size, rank, intermediate_size=intermediate_size + ) + assert layer.quant_method.weight_scale_refine is None + assert layer.quant_method.moe_block_shape == [128, 128] + width = layer.moe_config.intermediate_size_per_partition + assert width == ((num_blocks + tp_size - 1) // tp_size) * 128 + for name in weights: + prefix = "w2" if name == "w2" else "w13" + for suffix, checkpoint in ( + ("weight", weights[name]), + ("weight_scale_inv", scales[name]), + ): + param_name = f"{prefix}_{suffix}" + param = getattr(layer, param_name) + # Also reload after dirtying the destination, to catch padding leaks. + for _ in range(2): + if name != "w3": + param.data.fill_(7) + if batched: + assert param.weight_loader( + param, checkpoint, param_name, name, 0, return_success=True + ) + else: + for expert in range(2): + assert param.weight_loader( + param, + checkpoint[expert], + param_name, + name, + expert, + return_success=True, + ) + w = getattr(layer, f"{prefix}_weight").float() + s = getattr(layer, f"{prefix}_weight_scale_inv") + if name in ("w1", "w3"): + half = 0 if name == "w1" else 1 + w = w.chunk(2, dim=1)[half] + s = s.chunk(2, dim=1)[half] + dim = 2 if name == "w2" else 1 + valid = max(0, min(width, intermediate_size - rank * width)) + assert torch.count_nonzero(w.narrow(dim, valid, width - valid)) == 0 + assert torch.all(s.narrow(dim, valid // 128, (width - valid) // 128) == 1) + dequant = w * s.repeat_interleave(128, dim=1).repeat_interleave(128, dim=2) + reconstructed[name].append(dequant.narrow(dim, 0, valid)) + for name, parts in reconstructed.items(): + expected = weights[name].float() * scales[name].repeat_interleave( + 128, dim=1 + ).repeat_interleave(128, dim=2) + assert torch.equal(torch.cat(parts, dim=2 if name == "w2" else 1), expected) + + +@pytest.mark.parametrize("backend", ["auto", "triton"]) +@pytest.mark.parametrize("tp_size,block", [(2, 64), (4, 32)]) +def test_fp8_tp_default_keeps_refined_layout(monkeypatch, backend, tp_size, block): + layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0, backend) + assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size + assert layer.quant_method.moe_block_shape == [block, block] + + +@pytest.mark.parametrize( + "field,value", + [ + ("hidden_dim", 192), + ("intermediate_size", 672), + ("is_lora_enabled", True), + ("has_bias", True), + ], +) +def test_fp8_block_aligned_tp_rejects_unsupported_layout(monkeypatch, field, value): + with pytest.raises(ValueError, match="Block-aligned FP8 TP sharding requires"): + _make_fp8_tp_experts(monkeypatch, 4, 0, config_overrides={field: value}) + + +@pytest.mark.parametrize("tp_size", [1, 5]) +def test_fp8_aligned_tp_keeps_original_layout(monkeypatch, tp_size): + layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0) + assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size + assert not layer.moe_config.tp_shard_with_padding + assert layer.quant_method.weight_scale_refine is None + + +def test_fp8_skipped_layer_keeps_original_tp_layout(monkeypatch, default_vllm_config): + layer = _make_fp8_tp_experts( + monkeypatch, + 4, + 0, + quant_config=Fp8Config( + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128], + ignored_layers=["model.layers.0.mlp.experts"], + ), + ) + assert isinstance(layer.quant_method, unquantized_module.UnquantizedFusedMoEMethod) + assert not layer.moe_config.tp_shard_with_padding + assert layer.moe_config.intermediate_size_per_partition == 160 + assert layer.w13_weight.shape == (2, 320, 256) + assert layer.w2_weight.shape == (2, 256, 160) + + +def test_fp8_block_aligned_tp_rejects_presharded_weights(monkeypatch): + layer = _make_fp8_tp_experts(monkeypatch, 4, 0) + with pytest.raises(ValueError, match="unsharded checkpoint"): + layer.w13_weight.weight_loader( + layer.w13_weight, + torch.zeros(160, 256, dtype=torch.float8_e4m3fn), + "w13_weight", + "w1", + 0, + ) + + +@pytest.mark.parametrize("tp_size", [2, 4, 8]) +@pytest.mark.parametrize("num_tokens", [1, 17]) +@torch.inference_mode() +def test_fp8_block_aligned_tp_flashinfer_matches_unsharded( + monkeypatch, + tp_size, + num_tokens, + dist_init, + workspace_init, +): + """Compare sharded FI with unsharded FI, reporting the cross-backend + difference separately because activation quantization can differ. + """ + from vllm.config import VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config + from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts + from vllm.platforms import current_platform + + if not current_platform.is_device_capability_family(100): + pytest.skip("Requires datacenter Blackwell and FlashInfer TRTLLM") + torch.manual_seed(42) + e, h, n, topk = 16, 2560, 640, 10 + with torch.device("cuda"), set_current_vllm_config(VllmConfig()): + x = torch.randn(num_tokens, h, dtype=torch.bfloat16) / 10 + logits = torch.randn(num_tokens, e, dtype=torch.float32) + probabilities = logits.softmax(dim=-1) + topk_weights, topk_ids = probabilities.topk(topk, dim=-1) + topk_weights /= topk_weights.sum(dim=-1, keepdim=True) + weights = { + "w1": torch.randn(e, n, h).to(torch.float8_e4m3fn), + "w3": torch.randn(e, n, h).to(torch.float8_e4m3fn), + "w2": torch.randn(e, h, n).to(torch.float8_e4m3fn), + } + scales = { + name: (torch.rand(e, w.shape[1] // 128, w.shape[2] // 128) + 0.5) / 32 + for name, w in weights.items() + } + ref = fused_experts( + x, + torch.cat([weights["w1"], weights["w3"]], dim=1), + weights["w2"], + topk_weights, + topk_ids, + quant_config=fp8_w8a8_moe_quant_config( + w1_scale=torch.cat([scales["w1"], scales["w3"]], dim=1), + w2_scale=scales["w2"], + block_shape=[128, 128], + ), + ) + total = torch.zeros_like(ref, dtype=torch.float32) + fi_reference = None + for rank in range(-1, tp_size): + layer = _make_fp8_tp_experts( + monkeypatch, + 1 if rank == -1 else tp_size, + max(rank, 0), + mock_backend=False, + num_experts=e, + hidden_dim=h, + ) + for name in weights: + prefix = "w2" if name == "w2" else "w13" + for suffix, checkpoint in ( + ("weight", weights[name]), + ("weight_scale_inv", scales[name]), + ): + param_name = f"{prefix}_{suffix}" + param = getattr(layer, param_name) + param.weight_loader(param, checkpoint, param_name, name, 0) + method = layer.quant_method + method.process_weights_after_loading(layer) + assert method.is_monolithic + out = method.apply_monolithic(layer, x, logits) + assert torch.isfinite(out).all() + if rank == -1: + fi_reference = out.float().clone() + else: + total += out.float() + # Different FP8 GEMM reduction orders need not be bitwise identical. + assert fi_reference is not None + relative_l2 = (total - fi_reference).norm() / fi_reference.norm().clamp_min( + 1e-8 + ) + cross_backend_l2 = ( + fi_reference - ref.float() + ).norm() / ref.float().norm().clamp_min(1e-8) + print( + f"TP{tp_size} M={num_tokens}: shard_l2={relative_l2.item():.6f}, " + f"unsharded_fi_vs_triton_l2={cross_backend_l2.item():.6f}" + ) + assert relative_l2 < 0.01, relative_l2.item() diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 9a2f8229b1..35973e635c 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -1309,6 +1309,8 @@ class FusedMoEConfig: # Set by __post_init__ intermediate_size_per_partition: int = -1 + # Use the allocated width as the checkpoint TP stride, including for scales. + tp_shard_with_padding: bool = False rocm_aiter_fmoe_enabled: bool = False aiter_fmoe_shared_expert_enabled: bool = False diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 7c57e2e5c2..6a6ce34d16 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -31,11 +31,14 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( prepare_fp8_moe_layer_for_marlin, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, QuantKey, + create_fp8_quant_key, kFp8Dynamic128Sym, kFp8Static128BlockSym, ) from vllm.platforms import current_platform +from vllm.utils.math_utils import round_up logger = init_logger(__name__) @@ -299,6 +302,80 @@ def refine_fp8_moe_block_shape( return [refine, refine] +def pad_tp_shard_to_weight_blocks( + config: FusedMoEConfig, + weight_block_size: list[int], +) -> bool: + """Pad the TP shard to whole checkpoint blocks, keeping scales rank-local.""" + block_n, block_k = weight_block_size + if ( + block_n != block_k + or config.tp_size == 1 + or config.intermediate_size_per_partition % block_n == 0 + ): + return False + if ( + config.intermediate_size % block_n != 0 + or config.hidden_dim % block_n != 0 + or config.ep_size != 1 + or config.is_lora_enabled + or config.has_bias + ): + raise ValueError( + f"Block-aligned FP8 TP sharding requires {block_n}-aligned " + "global expert dimensions, pure TP, and no LoRA or " + "expert bias." + ) + config.tp_shard_with_padding = True + config.intermediate_size_per_partition = round_up( + config.intermediate_size_per_partition, block_n + ) + return True + + +def resolve_fp8_moe_weight_block_shape( + config: FusedMoEConfig, + weight_block_size: list[int], + activation_key: QuantKey, + is_checkpoint_fp8_serialized: bool, +) -> tuple[list[int], tuple[int, int] | None]: + """Return the TP-adapted block shape and refine factor: + refine if kernels allow, else pad to the TP shard.""" + refined_shape = refine_fp8_moe_block_shape(config, weight_block_size) + if is_checkpoint_fp8_serialized and config.moe_backend != "auto": + kernel_classes = backend_to_kernel_cls(map_fp8_backend(config.moe_backend)) + can_refine = refined_shape is not None and any( + k_cls._supports_quant_scheme( + create_fp8_quant_key( + static=True, group_shape=GroupShape(*refined_shape) + ), + activation_key, + ) + for k_cls in kernel_classes + ) + if not can_refine and pad_tp_shard_to_weight_blocks(config, weight_block_size): + logger.info_once( + "FP8 %s TP loading uses complete checkpoint blocks: " + "local allocation %d, without weight requantization.", + config.moe_backend, + config.intermediate_size_per_partition, + ) + return weight_block_size, None + if refined_shape is None: + return weight_block_size, None + logger.info_once( + "FP8 MoE block scales refined from %s to %s to fit " + "the TP-sharded intermediate size %d.", + str(weight_block_size), + str(refined_shape), + config.intermediate_size_per_partition, + ) + return refined_shape, ( + weight_block_size[0] // refined_shape[0], + weight_block_size[1] // refined_shape[1], + ) + + def select_fp8_moe_backend( config: FusedMoEConfig, weight_key: QuantKey | None, diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 88841070f0..42ea15dcda 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -332,6 +332,7 @@ class RoutedExperts(PluggableLayer): shard_id: str, loaded_weight: torch.Tensor, tp_rank: int, + is_scale: bool = False, ): """Load grouped weight scales for group quantization or model weights @@ -341,14 +342,41 @@ class RoutedExperts(PluggableLayer): shard_id: either w1, w2, or w3 loaded_weight: checkpoint weight to load into the param tp_rank: tensor parallel rank + is_scale: whether padding should use unit scales instead of zero weights. """ + padded_tp = self.moe_config.tp_shard_with_padding + if padded_tp: + destination = expert_data + if shard_id in ("w1", "w3") and self.moe_config.is_act_and_mul: + half = destination.shape[shard_dim] // 2 + destination = destination.narrow( + shard_dim, 0 if shard_id == "w1" else half, half + ) + shard_size = destination.shape[shard_dim] + expected_size = ( + self.moe_config.intermediate_size + * shard_size + // self.moe_config.intermediate_size_per_partition + ) + if loaded_weight.shape[shard_dim] != expected_size: + raise ValueError( + "Block-aligned TP loading expects an unsharded checkpoint " + f"projection of size {expected_size}, got " + f"{loaded_weight.shape[shard_dim]}." + ) + start = min(tp_rank * shard_size, expected_size) + size = min(shard_size, expected_size - start) + loaded_weight = loaded_weight.narrow(shard_dim, start, size) + destination.fill_(1 if is_scale else 0) + if shard_id == "w2": self._load_w2( shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=tp_rank, + load_full=padded_tp, ) elif shard_id in ("w1", "w3"): self._load_w13( @@ -357,6 +385,7 @@ class RoutedExperts(PluggableLayer): loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=tp_rank, + load_full=padded_tp, ) def _load_per_channel_weight_scale( @@ -511,11 +540,12 @@ class RoutedExperts(PluggableLayer): shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int, + load_full: bool = False, ): # Index the loaded weight for tp sharding. # down_proj: "RowParallel" so tp sharding on input_dim - # Only narrow if the loaded_weight is not a scalar (0-dim tensor). - if loaded_weight.ndim > 0: + # Padded TP weights have already been sliced by the grouped loader. + if not load_full and loaded_weight.ndim > 0: # Same padding fix as _load_w13: use unpadded per-rank size. tp_size = self.moe_config.moe_parallel_config.tp_size loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size @@ -800,6 +830,7 @@ class RoutedExperts(PluggableLayer): loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=self.moe_config.tp_rank, + is_scale=True, ) elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value: self._load_per_tensor_weight_scale( diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 5835af111a..c85a2c288e 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -8,7 +8,6 @@ import torch import vllm.envs as envs from vllm.config import get_current_vllm_config from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( init_fp8_linear_kernel, ) @@ -24,14 +23,12 @@ from vllm.model_executor.layers.fused_moe import ( SharedExperts, UnquantizedFusedMoEMethod, ) -from vllm.model_executor.layers.fused_moe.config import ( - FusedMoEQuantConfig, -) +from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_kernel, make_fp8_moe_quant_config, - refine_fp8_moe_block_shape, + resolve_fp8_moe_weight_block_shape, select_fp8_moe_backend, ) from vllm.model_executor.layers.linear import ( @@ -90,8 +87,6 @@ if TYPE_CHECKING: ACTIVATION_SCHEMES = ["static", "dynamic"] -logger = init_logger(__name__) - class Fp8Config(QuantizationConfig): """Config class for FP8.""" @@ -511,32 +506,16 @@ class Fp8MoEMethod(FusedMoEMethodBase): # Set weight key and activation key for kernel compatibility if self.block_quant: assert self.weight_block_size is not None - # TP shards the intermediate dim of the expert weights, so a - # per-shard size that is not a multiple of the checkpoint's block - # size makes the checkpoint's block scales impossible to shard - # exactly. When a finer block size (>= 32) divides both the - # checkpoint blocks and all involved dims, the weight scales are - # refined to that granularity at load time (a lossless upsampling, - # since the refined block divides the checkpoint block). The - # refined block shape is encoded in the weight key, so the oracle - # only selects kernels that support it (e.g. Triton, which takes - # the block shape as a runtime argument). - refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size) - if refined_shape is not None: - block_n, block_k = self.weight_block_size - self.weight_scale_refine = ( - block_n // refined_shape[0], - block_k // refined_shape[1], + # Adapt the checkpoint block shape to TP sharding before + # building the weight key. + self.moe_block_shape, self.weight_scale_refine = ( + resolve_fp8_moe_weight_block_shape( + self.moe, + self.weight_block_size, + kFp8Dynamic128Sym, + self.quant_config.is_checkpoint_fp8_serialized, ) - self.moe_block_shape = refined_shape - logger.info_once( - "FP8 MoE block scales refined from %s to %s to fit " - "the TP-sharded intermediate size %d.", - str(self.weight_block_size), - str(refined_shape), - self.moe.intermediate_size_per_partition, - ) - assert self.moe_block_shape is not None + ) weight_key = create_fp8_quant_key( static=True, group_shape=GroupShape(*self.moe_block_shape) ) From 40b40d1d39bb82563dd699be3456abaa95d0722a Mon Sep 17 00:00:00 2001 From: drakosha Date: Thu, 17 Sep 2026 13:19:56 +0300 Subject: [PATCH 013/143] [Bugfix][KV Offload] Register the offload region in chunks (#51081) Signed-off-by: Mikhail Kostryukov Co-authored-by: Claude Co-authored-by: Colton Ottley Co-authored-by: Or Ozeri --- .../kv_offload/cpu/test_canonical_layout.py | 1 + tests/v1/kv_offload/cpu/test_gpu_worker.py | 16 ++++ .../cpu/test_shared_offload_region.py | 82 +++++++++++++++++ .../device_communicators/cuda_wrapper.py | 26 ++++++ vllm/v1/kv_offload/cpu/gpu_worker.py | 88 +++++++++++++++---- 5 files changed, 198 insertions(+), 15 deletions(-) diff --git a/tests/v1/kv_offload/cpu/test_canonical_layout.py b/tests/v1/kv_offload/cpu/test_canonical_layout.py index 4594e1eb09..749bd6fac5 100644 --- a/tests/v1/kv_offload/cpu/test_canonical_layout.py +++ b/tests/v1/kv_offload/cpu/test_canonical_layout.py @@ -241,6 +241,7 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int): # The Triton load path dereferences CPU pointers on the GPU, which is # only legal on pinned memory; production pins via CPUOffloadingWorker pin_mmap_region(region) + assert region.is_pinned return region.create_next_canonical_view(_CANONICAL_PAGE) try: diff --git a/tests/v1/kv_offload/cpu/test_gpu_worker.py b/tests/v1/kv_offload/cpu/test_gpu_worker.py index 5306d05571..731d61a7e9 100644 --- a/tests/v1/kv_offload/cpu/test_gpu_worker.py +++ b/tests/v1/kv_offload/cpu/test_gpu_worker.py @@ -49,6 +49,22 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None: ) +@pytest.mark.skipif(not gpu_worker.HAS_TRITON, reason="Requires Triton") +def test_unpinned_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None: + """The Triton load path dereferences CPU pointers on the GPU, so pageable + host memory takes the DMA path even for pages where Triton would win.""" + monkeypatch.setattr(gpu_worker.current_platform, "is_xpu", lambda: False) + monkeypatch.setattr(gpu_worker.current_platform, "is_rocm", lambda: False) + + refs = [[CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=512)]] + assert gpu_worker._select_swap_blocks_fn(refs, gpu_to_cpu=False) is not ( + ops.swap_blocks_batch + ) + assert gpu_worker._select_swap_blocks_fn( + refs, gpu_to_cpu=False, host_memory_is_pinned=False + ) is (ops.swap_blocks_batch) + + def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None: """Both directions drain before the worker releases its shared region.""" worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker) diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index 4f22d122db..f84e1bd729 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -15,6 +15,7 @@ from unittest.mock import MagicMock, call import pytest from vllm.utils.system_utils import get_mp_context +from vllm.v1.kv_offload.cpu import gpu_worker from vllm.v1.kv_offload.cpu import shared_offload_region as region_module from vllm.v1.kv_offload.cpu.shared_offload_region import ( SharedOffloadRegion, @@ -788,6 +789,87 @@ def test_cleanup_unregisters_every_pinned_chunk(iid, monkeypatch): ] +@pytest.fixture +def host_register(monkeypatch): + """Pretend to run on CUDA with host registrations capped at seven pages. + + Registration goes to a mock CudaRTLibrary and the region's cleanup to a + separate mock torch runtime. Both stay installed through the cleanup that + ``_region`` runs on exit. + """ + cudart = MagicMock() + cudart.cudaHostRegister.return_value = 0 + cudart.cudaHostUnregister.return_value = 0 + cudart.cudaGetLastError.return_value = 2 + torch_cudart = MagicMock() + torch_cudart.cudaHostUnregister.return_value = MagicMock(value=0) + monkeypatch.setattr(gpu_worker.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(gpu_worker, "MAX_HOST_REGISTER_CHUNK_BYTES", 7 * PAGE_SIZE) + monkeypatch.setattr(gpu_worker, "CudaRTLibrary", lambda: cudart) + monkeypatch.setattr(region_module.torch.cuda, "cudart", lambda: torch_cudart) + return cudart, torch_cudart + + +def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register): + """Chunks end on row boundaries: 3-page rows under a 7-page cap register + as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1.""" + cudart, torch_cudart = host_register + with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region: + gpu_worker.pin_mmap_region(region) + base = region._base.data_ptr() + assert cudart.mock_calls == [ + call.cudaHostRegister(base, 6 * PAGE_SIZE), + call.cudaHostRegister(base + 6 * PAGE_SIZE, 6 * PAGE_SIZE), + call.cudaHostRegister(base + 12 * PAGE_SIZE, 3 * PAGE_SIZE), + ] + assert region.is_pinned + + assert torch_cudart.cudaHostUnregister.call_args_list == [ + call(base + 12 * PAGE_SIZE), + call(base + 6 * PAGE_SIZE), + call(base), + ] + + +@pytest.mark.parametrize("fail_at", [0, 1, 2]) +def test_pin_mmap_region_failure_leaves_region_pageable(iid, host_register, fail_at): + """A failed chunk drains its pending error and unregisters the chunks + before it on the same handle, so the region is pinned whole or not at all.""" + cudart, torch_cudart = host_register + cudart.cudaHostRegister.side_effect = [0] * fail_at + [2] + with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region: + gpu_worker.pin_mmap_region(region) + base = region._base.data_ptr() + assert not region.is_pinned + assert region.pinned_addresses == [] + + registered = [base + i * 6 * PAGE_SIZE for i in range(fail_at)] + failed_size = min(6, 15 - 6 * fail_at) * PAGE_SIZE + assert cudart.mock_calls == [ + *(call.cudaHostRegister(address, 6 * PAGE_SIZE) for address in registered), + call.cudaHostRegister(base + fail_at * 6 * PAGE_SIZE, failed_size), + call.cudaGetLastError(), + *(call.cudaHostUnregister(address) for address in reversed(registered)), + ] + torch_cudart.cudaHostUnregister.assert_not_called() + + +def test_pin_mmap_region_without_cuda_runtime_stays_pageable( + iid, host_register, monkeypatch +): + """A CUDA runtime that cannot be loaded leaves the region pageable instead + of failing worker startup.""" + + def missing_runtime(): + raise AssertionError("libcudart is not loaded in the current process") + + monkeypatch.setattr(gpu_worker, "CudaRTLibrary", missing_runtime) + with _region(iid) as region: + gpu_worker.pin_mmap_region(region) + assert not region.is_pinned + assert region.pinned_addresses == [] + + def test_cleanup_after_create_next_worker_view_releases_mmap(iid): """cleanup() must close the mmap even after create_next_worker_view was called. create_next_worker_view returns a view that shares storage with _base; both must be diff --git a/vllm/distributed/device_communicators/cuda_wrapper.py b/vllm/distributed/device_communicators/cuda_wrapper.py index a5026163e9..d27d84c030 100644 --- a/vllm/distributed/device_communicators/cuda_wrapper.py +++ b/vllm/distributed/device_communicators/cuda_wrapper.py @@ -78,6 +78,16 @@ class CudaRTLibrary: cudaError_t, [ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint], ), + # cudaError_t cudaHostRegister ( void* ptr, size_t size, unsigned int flags ) + Function( + "cudaHostRegister", + cudaError_t, + [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint], + ), + # cudaError_t cudaHostUnregister ( void* ptr ) + Function("cudaHostUnregister", cudaError_t, [ctypes.c_void_p]), + # cudaError_t cudaGetLastError ( void ) + Function("cudaGetLastError", cudaError_t, []), ] # https://rocm.docs.amd.com/projects/HIPIFY/en/latest/tables/CUDA_Runtime_API_functions_supported_by_HIP.html # noqa @@ -92,6 +102,9 @@ class CudaRTLibrary: "cudaMemcpy": "hipMemcpy", "cudaIpcGetMemHandle": "hipIpcGetMemHandle", "cudaIpcOpenMemHandle": "hipIpcOpenMemHandle", + "cudaHostRegister": "hipHostRegister", + "cudaHostUnregister": "hipHostUnregister", + "cudaGetLastError": "hipGetLastError", } # class attribute to store the mapping from the path to the library @@ -185,3 +198,16 @@ class CudaRTLibrary: ) ) return devPtr + + def cudaHostRegister(self, ptr: int, size: int, flags: int = 0) -> int: + """Return the raw error code instead of raising, since callers can + fall back to pageable memory. Drain a failure with cudaGetLastError.""" + return self.funcs["cudaHostRegister"](ctypes.c_void_p(ptr), size, flags) + + def cudaHostUnregister(self, ptr: int) -> int: + """Return the raw error code instead of raising.""" + return self.funcs["cudaHostUnregister"](ctypes.c_void_p(ptr)) + + def cudaGetLastError(self) -> int: + """Return and clear the error pending on this thread.""" + return self.funcs["cudaGetLastError"]() diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index 037e3631e4..797abed5ea 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -11,6 +11,7 @@ import numpy as np import torch from vllm import _custom_ops as ops +from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON, triton @@ -38,10 +39,13 @@ logger = init_logger(__name__) def _select_swap_blocks_fn( layer_refs_per_group: list[list[CanonicalKVCacheRef]], gpu_to_cpu: bool, + host_memory_is_pinned: bool = True, ): """Resolve the swap_blocks function for a handler at init time.""" # GPU->CPU is bandwidth-bound; the dedicated copy engine beats Triton. - if gpu_to_cpu: + # The Triton kernel dereferences CPU pointers on the GPU, which is only + # valid for pinned host memory. + if gpu_to_cpu or not host_memory_is_pinned: return ops.swap_blocks_batch # Fall back to the C++ DMA path on platforms where Triton isn't usable # (e.g. ROCm host mappings) or where GPU kernels cannot directly @@ -190,8 +194,12 @@ def _canonical_block_sizes( return canonical_bytes_per_block +# Bound registration size to avoid driver limits on large host allocations. +MAX_HOST_REGISTER_CHUNK_BYTES = 64 * 1024**3 + + def pin_mmap_region(region: SharedOffloadRegion) -> None: - """Register the entire mmap as CUDA pinned memory via cudaHostRegister.""" + """Register row-aligned chunks, rolling back on failure.""" if not current_platform.is_cuda_alike(): logger.info( "Skipping mmap host registration on %s; cudaHostRegister is only " @@ -201,23 +209,65 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None: return rank = region.rank + try: + cudart = CudaRTLibrary() + except (AssertionError, AttributeError, OSError): + logger.warning( + "Could not load the CUDA runtime for host registration on rank=%d; " + "the offload region stays pageable", + rank, + exc_info=True, + ) + return base_ptr = region._base.data_ptr() - result = torch.cuda.cudart().cudaHostRegister(base_ptr, region.total_size_bytes, 0) - if result.value != 0: + total_size = region.total_size_bytes + # Chunks end on block-row boundaries, which are page aligned, so neither the + # driver's page rounding nor any single block transfer straddles two + # registrations. + rows_per_chunk = max(MAX_HOST_REGISTER_CHUNK_BYTES // region._row_stride, 1) + chunk_size = rows_per_chunk * region._row_stride + + # Register, drain and roll back through the same runtime handle, so a + # failed chunk leaves neither a pending error nor a partly pinned region. + addresses: list[int] = [] + for offset in range(0, total_size, chunk_size): + address = base_ptr + offset + size = min(chunk_size, total_size - offset) + result = cudart.cudaHostRegister(address, size) + if result == 0: + addresses.append(address) + continue + cudart.cudaGetLastError() logger.warning( - "cudaHostRegister failed for rank=%d (code=%d) — " - "transfers will still work but may be slower (unpinned DMA)", + "cudaHostRegister failed for rank=%d at %.2f of %.2f GB (code=%d); " + "the offload region stays pageable", rank, + offset / 1e9, + total_size / 1e9, result, ) - else: - logger.debug( - "cudaHostRegister rank=%d %.2f GB", - rank, - region.total_size_bytes / 1e9, - ) - region.is_pinned = True + for registered in reversed(addresses): + unregister_result = cudart.cudaHostUnregister(registered) + if unregister_result != 0: + cudart.cudaGetLastError() + logger.warning( + "cudaHostUnregister failed for rank=%d at %#x (code=%d); " + "that chunk stays registered until the process exits", + rank, + registered, + unregister_result, + ) + return + + region.pinned_addresses.extend(addresses) + region.is_pinned = True + logger.debug( + "cudaHostRegister rank=%d %.2f GB in %d chunk(s)", + rank, + total_size / 1e9, + len(addresses), + ) def _new_descriptor_buffers( @@ -248,6 +298,7 @@ class SingleDirectionOffloadingHandler: layer_refs_per_group: list[list[CanonicalKVCacheRef]], gpu_to_cpu: bool, canonical_layout: bool = False, + host_memory_is_pinned: bool = True, ): """Initialize a SingleDirectionOffloadingHandler. @@ -262,6 +313,8 @@ class SingleDirectionOffloadingHandler: gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU. canonical_layout: if True, CPU pages use the canonical layout described by the refs' mappings. + host_memory_is_pinned: whether the CPU tensors are pinned, so GPU + kernels may dereference them directly. """ assert len(gpu_tensors) == len(cpu_tensors) @@ -299,7 +352,7 @@ class SingleDirectionOffloadingHandler: self.gpu_to_cpu: bool = gpu_to_cpu self.layer_refs_per_group = layer_refs_per_group self._swap_blocks_batch = _select_swap_blocks_fn( - layer_refs_per_group, gpu_to_cpu + layer_refs_per_group, gpu_to_cpu, host_memory_is_pinned ) # GPU blocks may be smaller @@ -646,7 +699,7 @@ class SingleDirectionOffloadingHandler: last_event = last_transfer.end_event # assure job will start only after the previous one completes stream.wait_event(last_event) - # CPU->GPU reads from host pinned memory, which is never written + # CPU->GPU reads from host memory, which is never written # by a concurrent GPU stream, so CU_MEMCPY_SRC_ACCESS_ORDER_ANY is # safe and lets the driver pipeline source reads. GPU->CPU reads # from the live GPU KV cache, which the compute stream keeps @@ -764,6 +817,9 @@ class CPUOffloadingWorker(OffloadingWorker): logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors)) if mmap_region is not None and pin_memory: pin_mmap_region(mmap_region) + host_memory_is_pinned = pin_memory and ( + mmap_region is None or mmap_region.is_pinned + ) canonical_bytes_per_block = ( _canonical_block_sizes(kv_caches.group_data_refs, len(kv_caches.tensors)) @@ -813,6 +869,7 @@ class CPUOffloadingWorker(OffloadingWorker): layer_refs_per_group=kv_caches.group_data_refs, gpu_to_cpu=True, canonical_layout=canonical_layout, + host_memory_is_pinned=host_memory_is_pinned, ) self._load_handler = SingleDirectionOffloadingHandler( @@ -822,6 +879,7 @@ class CPUOffloadingWorker(OffloadingWorker): layer_refs_per_group=kv_caches.group_data_refs, gpu_to_cpu=False, canonical_layout=canonical_layout, + host_memory_is_pinned=host_memory_is_pinned, ) def submit_store( From b6e7c1f1f0430b5d4784aea391c42067581b7f76 Mon Sep 17 00:00:00 2001 From: Itay Etelis <92247226+Etelis@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:22:49 +0300 Subject: [PATCH 014/143] [kv_offload] Skip scratch groups (#57145) Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Or Ozeri --- .../unit/offloading_connector/test_config.py | 33 +++++++++++++++ .../unit/offloading_connector/test_worker.py | 42 +++++++++++++++++++ .../kv_connector/v1/offloading/config.py | 6 +-- .../kv_connector/v1/offloading/worker.py | 5 ++- 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_config.py b/tests/v1/kv_connector/unit/offloading_connector/test_config.py index acf55972df..fe0d258462 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_config.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_config.py @@ -19,6 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( from vllm.platforms import current_platform from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, HiddenStateCacheSpec, KVCacheConfig, @@ -340,6 +341,38 @@ def test_worker_kv_bytes_preserves_tensor_layout(packed: bool): assert offloading_config.cache.blocks_per_chunk == 2 +def test_offloading_skips_scratch_group(): + kv_cache_config = _make_mamba_hybrid_kv_cache_config() + kv_cache_config.kv_cache_groups.insert( + 1, + KVCacheGroupSpec( + ["scratch"], + CircularBufferSpec( + block_size=4, num_kv_heads=1, head_size=128, dtype=torch.float32 + ), + ), + ) + page_size = kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes + kv_cache_config.kv_cache_tensors = [ + KVCacheTensor( + size=page_size * kv_cache_config.num_blocks, + layers=["full_layer", "scratch", "mamba_layer"], + layer_stride=0, + block_stride=page_size, + ) + ] + config = _make_vllm_config() + + offloading_config = build_offloading_config(config, kv_cache_config) + scheduler_config = SchedulerOffloadConfig.from_spec( + MockOffloadingSpec(offloading_config), config, kv_cache_config + ) + + assert [group.group_id for group in offloading_config.groups] == [0, 2] + assert [group.group_idx for group in scheduler_config.kv_group_configs] == [0, 2] + assert offloading_config.worker_kv_bytes_per_block == page_size + + def test_hisparse_offloads_only_indexer_group(): source = KVCacheGroupSpec( ["source"], diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py index 64f481d02e..32bd4f3eca 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py @@ -13,6 +13,7 @@ from vllm.platforms import current_platform from vllm.utils.torch_utils import get_dtype_size from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, @@ -520,6 +521,47 @@ def test_register_kv_caches(backend): assert actual.mapping is not None +def test_register_packed_kv_caches_skips_scratch_group(): + attn_spec = FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + ) + page = attn_spec.page_size_bytes + layers = ["layer0", "layer1"] + kv_cache_config = KVCacheConfig( + num_blocks=NUM_BLOCKS, + kv_cache_tensors=[ + KVCacheTensor( + size=2 * page * NUM_BLOCKS, + layers=layers, + layer_stride=page, + block_stride=2 * page, + ) + ], + kv_cache_groups=[ + KVCacheGroupSpec(layers, attn_spec), + KVCacheGroupSpec( + ["scratch"], + CircularBufferSpec( + block_size=4, num_kv_heads=1, head_size=1, dtype=DTYPE + ), + ), + ], + ) + kv_caches = _allocate_kv_caches( + kv_cache_config, [], device=torch.device(f"{DEVICE_TYPE}:0") + ) + worker, spec = _make_worker(kv_cache_config) + worker.register_kv_caches(kv_caches) + + canonical = spec.get_worker.call_args[0][0] + assert len(canonical.tensors) == 1 + assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, 2 * page) + assert canonical.group_data_refs == [[CanonicalKVCacheRef(0, 2 * page)]] + + @pytest.mark.parametrize("backend", ATTN_BACKENDS) def test_register_kv_caches_uniform_type(backend): """Test register_kv_caches with UniformTypeKVCacheSpecs. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py index ca503d4e67..e20c1189af 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py @@ -35,7 +35,7 @@ if TYPE_CHECKING: def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]: if kv_cache_config.hisparse_host_num_blocks is None: - return tuple(range(len(kv_cache_config.kv_cache_groups))) + return kv_cache_config.prefix_cacheable_group_ids return tuple( group_id for group_id, group in enumerate(kv_cache_config.kv_cache_groups) @@ -135,12 +135,12 @@ def build_offloading_config( ) worker_kv_bytes_per_block = 0 - all_groups_selected = len(selected_groups) == len(kv_cache_config.kv_cache_groups) if ( - all_groups_selected + kv_cache_config.hisparse_host_num_blocks is None and kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors ): + # Scratch filtering must preserve the scheduler/worker allocation stride. # Every KVCacheTensor describes placement within the same backing allocation, # so its size is the total, not a per-tensor share. total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index 5b14e50dda..2bb00d88f0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -141,8 +141,9 @@ class OffloadingConnectorWorker: ), None, ) - if packed_layer_name is not None and len(selected_groups) == len( - kv_cache_config.kv_cache_groups + if ( + packed_layer_name is not None + and kv_cache_config.hisparse_host_num_blocks is None ): (tensor,) = tensors_per_block[packed_layer_name] num_blocks = tensor.shape[0] From 683f6c8edf9656da99926bbe5aae5ffffaa850b3 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 17 Sep 2026 03:33:44 -0700 Subject: [PATCH 015/143] [CI] Raise H200 LM Eval Large Models timeout to 120 min (#57335) Signed-off-by: khluu Co-authored-by: Kimi --- .buildkite/test_areas/lm_eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index e94cbe5dfc..3eb7f0a444 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -296,7 +296,7 @@ steps: - label: ":nvidia: (H200) LM Eval Large Models" key: lm-eval-large-models-8xh200 - timeout_in_minutes: 50 + timeout_in_minutes: 120 device: h200 optional: true num_devices: 8 From 99ea5f525da6b5428f2c5740f3cd15377f284c1d Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 17 Sep 2026 03:34:00 -0700 Subject: [PATCH 016/143] [CI] Raise DSv4-Flash disaggregated engine readiness timeout to 1800s (#57334) Signed-off-by: khluu Co-authored-by: Kimi --- .buildkite/test_areas/disaggregated.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 0fbb32f8f9..d217090bc5 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -287,7 +287,7 @@ steps: PREFILL_BLOCK_SIZE: "256" DECODE_BLOCK_SIZE: "256" MODEL_NAMES: "deepseek-ai/DeepSeek-V4-Flash" - VLLM_ENGINE_READY_TIMEOUT_S: "900" + VLLM_ENGINE_READY_TIMEOUT_S: "1800" VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--kv-cache-dtype,fp8" # The V4 indexer only supports block-outermost layouts; let it pick. PREFILLER_KV_LAYOUT: "" From 0d726275653800e82c324e1d32b074f782e89315 Mon Sep 17 00:00:00 2001 From: Tianyu Guo Date: Thu, 17 Sep 2026 18:45:52 +0800 Subject: [PATCH 017/143] [Perf][EPD] Batch image requests per encoder (#57095) Signed-off-by: Tianyu Guo Signed-off-by: Zhou ziheng Co-authored-by: Zhou ziheng Co-authored-by: OpenAI Codex --- .../disaggregated_encoder/README.md | 16 ++ .../disaggregated_encoder/disagg_epd_proxy.py | 126 +++++++--- .../unit/test_ec_example_connector.py | 15 ++ .../ec_connector/unit/test_epd_proxy_retry.py | 216 ++++++++++++++++-- .../unit/test_scheduler_nixl_producer.py | 4 +- .../ec_transfer/ec_connector/utils.py | 11 +- 6 files changed, 329 insertions(+), 59 deletions(-) diff --git a/examples/disaggregated/disaggregated_encoder/README.md b/examples/disaggregated/disaggregated_encoder/README.md index bd2a46abbe..71a0d0e72e 100644 --- a/examples/disaggregated/disaggregated_encoder/README.md +++ b/examples/disaggregated/disaggregated_encoder/README.md @@ -106,6 +106,22 @@ If you enable prefill instance (`--prefill-servers-urls` not disabled), you will | `--decode-servers-urls` | Comma-separated list of decode endpoints. Non-stream and stream paths both round-robin over this list. | | `--host`, `--port` | Bind address for the proxy itself (defaults: `0.0.0.0:8000`). | +The proxy batches images from the same user request assigned to the same encoder. +Set the proxy environment variable `ENCODER_MAX_BATCH_SIZE` to limit the number +of images per encoder subrequest. It defaults to `0` (unlimited); `1` sends each +image separately. Audio and video remain separate subrequests. + +For encoders with a smaller `--limit-mm-per-prompt` image limit than P/PD, set +`ENCODER_MAX_BATCH_SIZE` no higher than the smallest encoder image limit. +For example, for encoders configured with `--limit-mm-per-prompt '{"image": 2}'`: + +```bash +ENCODER_MAX_BATCH_SIZE=2 python disagg_epd_proxy.py \ + --encode-servers-urls "http://e1:8001,http://e2:8002" \ + --prefill-servers-urls disable \ + --decode-servers-urls "http://pd1:8003" +``` + Example usage: For E + PD setup: diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index c44c902690..3317b091fb 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -10,8 +10,8 @@ clusters: For MM input we: 1. Extract *every* image/audio/video item. - 2. Fire N concurrent requests to the encoder cluster - (one request per item, with **all text removed**). + 2. Send concurrent encoder requests with all text removed, + grouping images assigned to the same encoder. 3. Wait for all of them to succeed. 4. Forward the *original* request to a decode server. """ @@ -99,6 +99,11 @@ def validate_ec_consumer_routing( # Diagnostic switch: forward the original request to the decoder so the # only difference from the rewrite path is the rewrite itself. NO_REWRITE = False +# Maximum images per encoder subrequest; 0 leaves batches unlimited. +ENCODER_MAX_BATCH_SIZE = int(os.getenv("ENCODER_MAX_BATCH_SIZE", "0")) +if ENCODER_MAX_BATCH_SIZE < 0: + raise ValueError("ENCODER_MAX_BATCH_SIZE must be non-negative") + # Decode-side retries for a retryable internal error (`finish_reason="error"`, # e.g. an encoder embedding the connector could not deliver). Re-issuing runs # the encode again, which produces a fresh transfer. @@ -222,7 +227,7 @@ async def fanout_encoder_primer( req_id: str, consumer_zmq: str | None = None, ) -> tuple[dict[int, dict], dict[str, Any]]: - """1. Build one request *per MM item* with all text removed. + """1. Group images by encoder, retaining per-item round-robin assignment. 2. Send them concurrently to the encode cluster. 3. Raise if any of them fails. @@ -248,6 +253,7 @@ async def fanout_encoder_primer( item_uuids: dict[int, str] = {} item_transfer_ids: dict[int, str] = {} item_meta: dict[int, dict] = {} + transfer_items: dict[int, dict[str, str]] = {} ec_params: dict[str, Any] = {} # Round-robin over encode servers to distribute load a bit. The cursor @@ -259,29 +265,45 @@ async def fanout_encoder_primer( e_urls, encoder_rr_idx, len(mm_items) ) + groups: list[tuple[str, list[int]]] = [] + image_groups: dict[str, list[int]] = {} for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)): + if item["type"] == "image_url": + indices = image_groups.get(target_url) + if indices is None or ( + ENCODER_MAX_BATCH_SIZE and len(indices) >= ENCODER_MAX_BATCH_SIZE + ): + indices = [] + image_groups[target_url] = indices + groups.append((target_url, indices)) + indices.append(idx) + else: + groups.append((target_url, [idx])) + + for target_url, indices in groups: # Derive a *child* request id: :: - child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}" + child_req_id = f"{req_id}:{indices[0]}:{uuid.uuid4().hex[:6]}" headers = {"x-request-id": child_req_id, "Content-Type": "application/json"} # With --no-rewrite the decoder still receives the raw image and derives # the cache key by hashing it, so the encoder must do the same -- passing # a uuid here would make the two disagree and silently defeat the EC # transfer, leaving the decoder to encode the image itself. - item_uuid = None if NO_REWRITE else content_uuid(item) - if item_uuid is not None: - item_uuids[idx] = item_uuid - transfer_id = uuid.uuid4().hex - item_transfer_ids[idx] = transfer_id + content = [] + for idx in indices: + item = mm_items[idx] + item_uuid = None if NO_REWRITE else content_uuid(item) + if item_uuid is not None: + item_uuids[idx] = item_uuid + item_transfer_ids[idx] = uuid.uuid4().hex + content.append(item if item_uuid is None else {**item, "uuid": item_uuid}) encoder_req = { "model": orig_request.get("model"), "messages": [ { "role": "user", - "content": [ - item if item_uuid is None else {**item, "uuid": item_uuid} - ], + "content": content, }, ], # No max_tokens cap: the encoder instance never samples, it finishes @@ -297,17 +319,13 @@ async def fanout_encoder_primer( if key in orig_request: encoder_req[key] = orig_request[key] if consumer_zmq is not None: - # No mm_hash here on purpose. The encoder's own - # `mm_features[i].identifier` is derived from the uuid *and* the - # engine's media_io_kwargs / mm_processor_kwargs, so this proxy - # cannot know it before the encoder runs. Sending the bare uuid - # would fail the connector's hash match and make the producer - # invent its own transfer id, which the consumer could then never - # cancel. Omitting it lets the connector match by position, which - # is exact: one encoder request carries exactly one item. + # The engine may rehash UUIDs with processing options. Match by + # position instead; batches contain only images in input order. encoder_req["ec_transfer_params"] = { "consumer_zmq": consumer_zmq, - "ec_items": [{"transfer_id": transfer_id}], + "ec_items": [ + {"transfer_id": item_transfer_ids[idx]} for idx in indices + ], } tasks.append( encode_session.post( @@ -320,7 +338,8 @@ async def fanout_encoder_primer( results = await asyncio.gather(*tasks, return_exceptions=True) # Fail fast if any sub-request failed - for idx, r in enumerate(results): + for (_, indices), r in zip(groups, results): + idx = indices[0] if isinstance(r, Exception): logger.error( "[%s] Encoder request #%d raised exception: %s", @@ -357,18 +376,45 @@ async def fanout_encoder_primer( logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx) params = {} if params: - # One encoder request carries exactly one item, so there is a - # single reported entry. Do not key it by this proxy's uuid: when - # media_io_kwargs or mm_processor_kwargs are set the engine - # re-hashes the uuid together with them, so the encoder's own - # `mm_features[i].identifier` is a derived value this proxy cannot - # predict. Fall back to the sole entry, and carry the key the - # encoder actually used through as `ec_mm_hash`. - ec_mm_hash = item_uuids.get(idx) - reported = params.get(ec_mm_hash) - if reported is None and len(params) == 1: - ((ec_mm_hash, reported),) = params.items() - if reported: + by_index = {} + for mm_hash, reported in params.items(): + if len(indices) == 1: + break + for local_index in reported.get("item_indices", []): + if type(local_index) is not int or not 0 <= local_index < len( + indices + ): + raise HTTPException(502, "Invalid encoder item index") + if local_index in by_index: + raise HTTPException(502, "Duplicate encoder item index") + by_index[local_index] = (mm_hash, reported) + for local_index, idx in enumerate(indices): + matched = by_index.get(local_index) + if matched is None: + # Compatibility with encoders without position metadata. + mm_hash = item_uuids.get(idx) + if mm_hash in params: + matched = (mm_hash, params[mm_hash]) + elif len(indices) == 1 and len(params) == 1: + matched = next(iter(params.items())) + elif ( + mm_items[idx]["type"] == "video_url" + and (orig_request.get("mm_processor_kwargs") or {}).get( + "use_audio_in_video" + ) + and len(params) == 2 + and consumer_zmq is None + and all( + entry.keys() <= {"metadata", "item_indices"} + for entry in params.values() + ) + ): + # Keep raw video for its audio/video features when no transfer + # handles need to be forwarded. + continue + else: + raise HTTPException(502, "Encoder metadata cannot be matched") + ec_mm_hash, reported = matched metadata = reported.get("metadata") or {} if metadata: item_meta[idx] = { @@ -384,12 +430,16 @@ async def fanout_encoder_primer( # looks it up by mm_hash on the request, so carry it through. ec_params[ec_mm_hash] = reported if NO_REWRITE and consumer_zmq is not None: - ec_params.setdefault("ec_items", []).append( - {"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]} - ) + transfer_items[idx] = { + "mm_hash": ec_mm_hash, + "transfer_id": item_transfer_ids[idx], + } + + if transfer_items: + ec_params["ec_items"] = [transfer_items[idx] for idx in sorted(transfer_items)] logger.info( - "[%s] All %d encoder requests completed successfully", req_id, len(mm_items) + "[%s] All %d encoder requests completed successfully", req_id, len(groups) ) return item_meta, ec_params diff --git a/tests/v1/ec_connector/unit/test_ec_example_connector.py b/tests/v1/ec_connector/unit/test_ec_example_connector.py index dac809d1a4..a6645fccf4 100644 --- a/tests/v1/ec_connector/unit/test_ec_example_connector.py +++ b/tests/v1/ec_connector/unit/test_ec_example_connector.py @@ -16,6 +16,7 @@ from vllm.distributed.ec_transfer.ec_connector.example_connector import ( ECExampleConnectorMetadata, MMMeta, ) +from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange from vllm.v1.core.sched.output import SchedulerOutput @@ -41,6 +42,20 @@ class MockRequest: return self._token_counts[input_id] +def test_metadata_preserves_repeated_item_positions(): + request = MockRequest("request", ["A", "B", "A"], [4, 4, 4]) + resolver = Mock() + resolver.fields_for.return_value = {"image_grid_thw"} + for feature in request.mm_features: + feature.data = Mock() + feature.data.get_data.return_value = {"image_grid_thw": torch.tensor([1, 2, 2])} + items = collect_ec_item_metadata(request.mm_features, resolver) + assert items == { + "A": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [0, 2]}, + "B": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [1]}, + } + + @pytest.fixture def temp_storage(tmp_path): """Fixture providing temporary storage path.""" diff --git a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py index 1a8c877c72..0c48dc84d6 100644 --- a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py +++ b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py @@ -9,6 +9,8 @@ change to them is what these tests catch. import asyncio import importlib.util from pathlib import Path +from types import SimpleNamespace +from typing import Any import httpx import msgspec @@ -83,6 +85,128 @@ class _EncoderSession: return _EncoderResponse(self._replies.pop(0)) +@pytest.mark.parametrize("no_rewrite", [False, True]) +@pytest.mark.parametrize( + "batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])] +) +def test_image_batches_preserve_item_identity( + proxy, monkeypatch, no_rewrite, batch_size, expected_sizes +): + """Rehashed and repeated images retain metadata and per-occurrence transfers.""" + from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import ( + ECMooncakeScheduler, + ) + + seen = [] + + class Session: + async def post(self, url, data=None, headers=None): + body = msgspec.json.decode(data) + seen.append((url, body)) + params: dict[str, dict[str, Any]] = {} + for index, item in enumerate(body["messages"][0]["content"]): + image = item["image_url"]["url"] + entry = params.setdefault( + image + "-processed", + { + "metadata": {"image_grid_thw": [1, 2, ord(image)]}, + "item_indices": [], + "peer_port": 1234, + }, + ) + entry["item_indices"].append(index) + # Response ordering must not affect the image-to-metadata mapping. + return _EncoderResponse(dict(reversed(list(params.items())))) + + monkeypatch.setattr(proxy, "encode_session", Session()) + monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite) + monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", batch_size) + monkeypatch.setattr(proxy, "encoder_rr_idx", 0) + images = ["A", "A", "C", "D", "A", "F"] + body = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": image}} + for image in images + ], + } + ], + "mm_processor_kwargs": {"max_pixels": 262144}, + } + meta, handles = asyncio.run( + proxy.fanout_encoder_primer( + body, ["http://e0", "http://e1"], "r", "tcp://consumer:1" + ) + ) + assert [len(b["messages"][0]["content"]) for _, b in seen] == expected_sizes + assert ( + len( + { + x["transfer_id"] + for _, b in seen + for x in b["ec_transfer_params"]["ec_items"] + } + ) + == 6 + ) + for url, batch in seen: + assert batch["mm_processor_kwargs"] == body["mm_processor_kwargs"] + for item in batch["messages"][0]["content"]: + assert item["image_url"]["url"] in ( + {"A", "C"} if url.startswith("http://e0/") else {"A", "D", "F"} + ) + assert [meta[i]["image_grid_thw"][-1] for i in range(6)] == list(map(ord, images)) + assert [meta[i]["ec_mm_hash"] for i in range(6)] == [ + x + "-processed" for x in images + ] + assert handles["A-processed"]["peer_port"] == 1234 + assert "ec_transfer_params" not in body + + transfers_by_encoder: list[list[dict[str, str]]] = [[], []] + for url, batch in seen: + rank = 0 if url.startswith("http://e0/") else 1 + transfers_by_encoder[rank].extend(batch["ec_transfer_params"]["ec_items"]) + expected_transfers = [ + transfers_by_encoder[i % 2][i // 2]["transfer_id"] for i in range(len(images)) + ] + params = ( + handles + if no_rewrite + else proxy.rewrite_for_decode(body, meta)["ec_transfer_params"] + ) + request = SimpleNamespace( + ec_transfer_params=params, + mm_features=[ + SimpleNamespace(identifier=image + "-processed") for image in images + ], + ) + assert [ + ECMooncakeScheduler._request_transfer_id(request, i) for i in range(len(images)) + ] == expected_transfers + + +def test_batch_rejects_ambiguous_metadata(proxy, monkeypatch): + monkeypatch.setattr( + proxy, + "encode_session", + _EncoderSession([{"unknown": {"metadata": {"image_grid_thw": [1, 2, 2]}}}]), + ) + body = { + "messages": [ + { + "content": [ + {"type": "image_url", "image_url": {"url": image}} + for image in ["A", "B"] + ] + } + ] + } + with pytest.raises(proxy.HTTPException, match="cannot be matched"): + asyncio.run(proxy.fanout_encoder_primer(body, ["http://e0"], "r")) + + def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch): """The retry loop re-enters `prepare_for_decode` with the same body. @@ -187,20 +311,21 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( body = await request.json() seen[stage].append(body) if stage == "encode": - item = body["messages"][0]["content"][0] - mm_hash = item["uuid"] - if body.get("mm_processor_kwargs"): - mm_hash += "-processed" - return web.json_response( - { - "ec_transfer_params": { - mm_hash: { - "metadata": {"image_grid_thw": [[1, 2, 2]]}, - "peer_port": len(seen[stage]), - } - } - } - ) + params: dict[str, dict[str, Any]] = {} + for index, item in enumerate(body["messages"][0]["content"]): + mm_hash = item["uuid"] + if body.get("mm_processor_kwargs"): + mm_hash += "-processed" + entry = params.setdefault( + mm_hash, + { + "metadata": {"image_grid_thw": [[1, 2, 2]]}, + "peer_port": len(seen[stage]), + "item_indices": [], + }, + ) + entry["item_indices"].append(index) + return web.json_response({"ec_transfer_params": params}) if stage == "prefill": assert body["max_tokens"] == 1 and body["stream"] is False return web.json_response( @@ -263,7 +388,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( finally: await proxy.on_shutdown() - assert len(seen["encode"]) == 4 + assert len(seen["encode"]) == 2 assert len(seen["decode"]) == 2 for requests in seen.values(): for forwarded in requests: @@ -296,7 +421,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( if request_options: mm_hash += "-processed" assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash} - assert final["ec_transfer_params"][mm_hash]["peer_port"] in (3, 4) + assert final["ec_transfer_params"][mm_hash]["peer_port"] == 2 if prefill: assert final["kv_transfer_params"] == {"remote_block_ids": [2]} @@ -324,3 +449,62 @@ def test_pooled_connections_are_retired_before_the_server_closes_them( assert proxy.decode_session.connector._keepalive_timeout == pooled_for finally: asyncio.run(proxy.on_shutdown()) + + +@pytest.mark.parametrize( + "no_rewrite, transfer", + [(False, None), (True, None), (False, "push"), (False, "handle")], +) +def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer): + """Keep raw video and neighboring images, but never discard transfer handles.""" + import copy + + from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata + + video = {"type": "video_url", "video_url": {"url": "video-with-audio"}} + images = [{"type": "image_url", "image_url": {"url": key}} for key in ("A", "B")] + body: dict[str, Any] = { + "messages": [{"content": [images[0], video, images[1]]}], + "mm_processor_kwargs": {"use_audio_in_video": True}, + } + original = copy.deepcopy(body) + # Collector indices describe two processed features from one video item. + video_metadata = collect_ec_item_metadata( + [SimpleNamespace(identifier=key, data=None) for key in ("video", "audio")], None + ) + if transfer == "handle": + video_metadata["audio"]["transfer_id"] = "reservation" + image_metadata = { + key: {"metadata": {"image_grid_thw": [1, 2, size]}, "item_indices": [i]} + for i, (key, size) in enumerate((("A", 2), ("B", 4))) + } + # Images form the first group, the singleton video the second. + monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", 0) + monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite) + replies = _EncoderSession([image_metadata, video_metadata]) + monkeypatch.setattr(proxy, "encode_session", replies) + consumer = "tcp://consumer:1" if transfer == "push" else None + preparation = proxy.prepare_for_decode(body, "r", ["http://e0"], "", consumer) + if transfer: + with pytest.raises(proxy.HTTPException, match="cannot be matched"): + asyncio.run(preparation) + return + + prepared, _, _ = asyncio.run(preparation) + assert body == original + assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"] + final = prepared["messages"][0]["content"] + assert final[1] == video + handles = prepared["ec_transfer_params"] + assert not video_metadata.keys() & handles.keys() + if no_rewrite: + assert final == original["messages"][0]["content"] + assert "ec_items" not in handles + else: + for position, image, key in ((0, images[0], "A"), (2, images[1], "B")): + assert final[position] == { + "type": "image_embeds", + "image_embeds": image_metadata[key]["metadata"], + "uuid": proxy.content_uuid(image), + } + assert [item["mm_hash"] for item in handles["ec_items"]] == ["A", "B"] diff --git a/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py b/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py index 156d98ae79..d7ef19b5f4 100644 --- a/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py +++ b/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py @@ -67,6 +67,7 @@ def test_request_finished_producer_emits_params(monkeypatch): assert params == { "h1": { "metadata": {}, + "item_indices": [0], "peer_host": "1.2.3.4", "peer_port": 5601, "size_bytes": 2 * 32 * 2, @@ -91,6 +92,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch): assert params == { "h1": { "metadata": {}, + "item_indices": [0], "peer_host": "1.2.3.4", "peer_port": 5601, "size_bytes": 2 * 32 * 2, @@ -110,5 +112,5 @@ def test_request_finished_skips_unallocated_entry(monkeypatch): assert delay is False # The item's placeholder metadata is still reported even though there's # no cache entry to transfer (empty "metadata": no fields, no transfer). - assert params == {"h1": {"metadata": {}}} + assert params == {"h1": {"metadata": {}, "item_indices": [0]}} s.shutdown() diff --git a/vllm/distributed/ec_transfer/ec_connector/utils.py b/vllm/distributed/ec_transfer/ec_connector/utils.py index f54d6877c1..35d26bb44b 100644 --- a/vllm/distributed/ec_transfer/ec_connector/utils.py +++ b/vllm/distributed/ec_transfer/ec_connector/utils.py @@ -60,15 +60,16 @@ def collect_ec_item_metadata( Keyed by mm_hash, each entry carries a `metadata` dict with whatever placeholder fields `resolver` says this model needs published for its - modality, so a consumer can skip the image transform. `data` is None for - items served from the processor cache, in which case the metadata is + modality, so a consumer can skip the image transform. `item_indices` + identifies every occurrence in `mm_features`, including repeated hashes. + `data` is None for items served from the processor cache, so the metadata is unavailable here and the consumer has to fall back to processing the media itself. A connector that also has transfer coordinates to report (e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside `metadata`, not into it. """ items: dict[str, dict[str, Any]] = {} - for feature in mm_features: + for index, feature in enumerate(mm_features): metadata: dict[str, Any] = {} if feature.data is not None: wanted = resolver.fields_for(feature.modality) @@ -81,7 +82,9 @@ def collect_ec_item_metadata( # Some metadata (e.g. Qwen3-VL video timestamps) is # produced as a plain list rather than a tensor. metadata[key] = value - items[feature.identifier] = {"metadata": metadata} + entry = items.setdefault(feature.identifier, {"item_indices": []}) + entry["metadata"] = metadata + entry["item_indices"].append(index) return items From 9ff08f9493aa745b0559c02cf6148f71d9817169 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Thu, 17 Sep 2026 18:46:28 +0800 Subject: [PATCH 018/143] [Bugfix] Honor skip_reading_prefix_cache for KV connector hits (#57269) Signed-off-by: aoshen02 Co-authored-by: aoshen02 --- tests/v1/core/test_scheduler.py | 19 +++++++++++++++++++ vllm/v1/core/sched/scheduler.py | 2 ++ 2 files changed, 21 insertions(+) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index ef818f27c7..298c8c6185 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -2302,6 +2302,25 @@ def test_has_sync_kv_loads( assert output.has_sync_kv_loads is expected_has_sync_loads +def test_kv_connector_honors_skip_reading_prefix_cache(): + """A request that must score every prompt row takes no external hit.""" + BLOCK_SIZE = 16 + scheduler = create_scheduler( + enable_prefix_caching=True, + use_kv_connector=mock_kv(matched_tokens=BLOCK_SIZE * 2, is_async=False), + block_size=BLOCK_SIZE, + ) + plain, scoring = create_requests(num_requests=2, num_tokens=BLOCK_SIZE * 4) + scoring.sampling_params.prompt_logprobs = 1 + scoring.skip_reading_prefix_cache = True + scheduler.add_request(plain) + scheduler.add_request(scoring) + + output = scheduler.schedule() + assert output.num_scheduled_tokens[plain.request_id] == BLOCK_SIZE * 2 + assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4 + + @pytest.mark.parametrize("is_async", [False, True]) def test_kv_connector_basic(is_async: bool): """Test whether Scheduler with KVConnector schedules tokens, allocates diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ed80daa8c2..e0c4997b63 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -935,6 +935,8 @@ class Scheduler(SchedulerInterface): request, block_aligned_local ) ) + if request.skip_reading_prefix_cache: + ext_tokens, load_kv_async = 0, False if ext_tokens is None: # The request cannot be scheduled because From da3c07bf7dda0c5cbe94d06854426907cd30c7ed Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Thu, 17 Sep 2026 19:57:22 +0800 Subject: [PATCH 019/143] [XPU][CI] skip test_hybrid_prefix_cache_hit_rate (#57301) Signed-off-by: Yan Ma Co-authored-by: Kunshang Ji --- .buildkite/intel_jobs/test-intel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index ef2b4ba073..458bef9518 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -98,7 +98,7 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && pip install dpctl && - pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py && + pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py --ignore=v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py && pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py && pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" && pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py && From 667b26e50bc3287793efd92672c520fa101ecf80 Mon Sep 17 00:00:00 2001 From: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:02:39 +0200 Subject: [PATCH 020/143] [Bugfix][ROCm][GLM-5.3-Flash] Apply deferred tilelang.jit already on attribute access (#57192) Signed-off-by: simondanielsson --- tests/kernels/test_mhc_tilelang_jit.py | 36 ++++++++++++++---- vllm/tilelang_utils/__init__.py | 51 ++++++++++++++++++-------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/tests/kernels/test_mhc_tilelang_jit.py b/tests/kernels/test_mhc_tilelang_jit.py index 4f39fae020..1c430e7ae3 100644 --- a/tests/kernels/test_mhc_tilelang_jit.py +++ b/tests/kernels/test_mhc_tilelang_jit.py @@ -24,19 +24,26 @@ class _PassConfigKey: def _install_tilelang_stub( monkeypatch: pytest.MonkeyPatch, ) -> dict[str, int]: - calls = {"jit_decorate": 0, "compiled_call": 0} + calls = {"jit_decorate": 0, "compiled_call": 0, "compiled_compile": 0} tilelang: Any = ModuleType("tilelang") + class _JitImpl: + def __init__(self, func: Any) -> None: + self.func = func + + def __call__(self, *args: Any, **kw: Any) -> Any: + calls["compiled_call"] += 1 + return self.func.__name__ + + def compile(self, *args: Any, **kw: Any) -> Any: + calls["compiled_compile"] += 1 + return self.func.__name__ + def jit(**kwargs: Any) -> Any: def decorate(func: Any) -> Any: calls["jit_decorate"] += 1 - - def compiled(*args: Any, **kw: Any) -> Any: - calls["compiled_call"] += 1 - return func.__name__ - - return compiled + return _JitImpl(func) return decorate @@ -78,6 +85,21 @@ def test_tilelang_jit_decorator_is_lazy_only_on_rocm( assert calls["compiled_call"] == 1 +@pytest.mark.skipif(not current_platform.is_rocm(), reason="Test requires ROCm") +def test_tilelang_jit_proxies_compile_only_warmup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = _install_tilelang_stub(monkeypatch) + module_name = "vllm.model_executor.kernels.mhc.tilelang_kernels" + monkeypatch.delitem(sys.modules, module_name, raising=False) + module = importlib.import_module(module_name) + + assert module.mhc_post_tilelang.compile() == "mhc_post_tilelang" + assert calls["jit_decorate"] == 1 + assert calls["compiled_compile"] == 1 + assert calls["compiled_call"] == 0 + + @pytest.mark.skipif(not current_platform.is_rocm(), reason="Test requires ROCm") def test_deepseek_v4_import_and_jit_monitor_do_not_hijack_hip_symbols() -> None: if importlib.util.find_spec("tilelang") is None: diff --git a/vllm/tilelang_utils/__init__.py b/vllm/tilelang_utils/__init__.py index 990c2fca3e..5ac03c858e 100644 --- a/vllm/tilelang_utils/__init__.py +++ b/vllm/tilelang_utils/__init__.py @@ -61,8 +61,41 @@ def _get_pass_configs() -> dict[Any, Any]: return pass_configs +class _DeferredTileLangJitKernel: + """Stand-in for a `tilelang.jit` kernel that decorates on first use. + + Both attribute access and calling apply the decoration. Required for + compile-only JIT warmup on platforms that defer import of tilelang. + """ + + _kernel_function: Callable[..., Any] | None = None + _jit_kernel: Any = None + + def __init__(self, kernel_function: Callable[..., Any]) -> None: + self._kernel_function = kernel_function + functools.update_wrapper(self, kernel_function) + + def _ensure_jit_kernel(self) -> Any: + if self._jit_kernel is None: + _ensure_tilelang_imported() + kernel_function = self._kernel_function + assert kernel_function is not None + kernel_function.__globals__["tilelang"] = tilelang + kernel_function.__globals__["T"] = T + self._jit_kernel = tilelang.jit(pass_configs=_get_pass_configs())( + kernel_function + ) + return self._jit_kernel + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return self._ensure_jit_kernel()(*args, **kwargs) + + def __getattr__(self, name: str) -> Any: + return getattr(self._ensure_jit_kernel(), name) + + def tilelang_jit(kernel_function: Callable[..., Any]) -> Callable[..., Any]: - """Apply `tilelang.jit`, deferring until first call on ROCm. + """Apply `tilelang.jit`, deferring until first use on ROCm. ROCm defers JIT decoration so importing the caller's module does not require TileLang immediately. CUDA keeps the eager decoration behavior. @@ -75,18 +108,4 @@ def tilelang_jit(kernel_function: Callable[..., Any]) -> Callable[..., Any]: _ensure_tilelang_imported() return tilelang.jit(pass_configs=_get_pass_configs())(kernel_function) - compiled_kernel: Callable[..., Any] | None = None - - @functools.wraps(kernel_function) - def wrapper(*args: Any, **kwargs: Any) -> Any: - nonlocal compiled_kernel - if compiled_kernel is None: - _ensure_tilelang_imported() - kernel_function.__globals__["tilelang"] = tilelang - kernel_function.__globals__["T"] = T - compiled_kernel = tilelang.jit(pass_configs=_get_pass_configs())( - kernel_function - ) - return compiled_kernel(*args, **kwargs) - - return wrapper + return _DeferredTileLangJitKernel(kernel_function) From 91a4c40b45696c4dd31281877ad46af0910efcfa Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 17 Sep 2026 08:04:56 -0400 Subject: [PATCH 021/143] [Compile] Fix compile warning #177-D (#56609) Signed-off-by: yewentao256 --- csrc/libtorch_stable/cache_kernels.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index 887c83f3af..c5e35c52b8 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -84,7 +84,9 @@ void swap_blocks(torch::stable::Tensor& src, torch::stable::Tensor& dst, namespace { // ROCm hipMemcpyBatchAsync faults for count > 8192 (MI350X/gfx950), so chunk // at that ceiling. +#if defined(USE_ROCM) constexpr int64_t kRocmDefaultMaxBatchDescriptors = 8192; +#endif constexpr const char* kMaxBatchDescriptorsEnv = "VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS"; From a85dd55177d59b99903432c71c7376309d7e7757 Mon Sep 17 00:00:00 2001 From: Laura Lopez-Real <113060513+laulopezreal@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:22:57 +0200 Subject: [PATCH 022/143] Fix Laguna patch mutating flat RoPE parameters (#57189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Laura López Real Co-authored-by: Laura López Real Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/test_config.py | 37 +++++++++++++++++++++++++++++++ vllm/transformers_utils/config.py | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 5c892da6c6..a07929abfc 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -46,6 +46,7 @@ from vllm.config.utils import get_field from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel from vllm.platforms import current_platform from vllm.transformers_utils.config import ( + _patch_hf_transformers_nested_rope_validation, get_pooling_config, try_get_dense_modules, ) @@ -54,6 +55,42 @@ from vllm.v1.attention.backend import AttentionCGSupport DEVICE_TYPE = current_platform.device_type +def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch): + calls = [] + + def original_validate_rope(config, *args, **kwargs): + calls.append(config) + + from transformers import PretrainedConfig + + monkeypatch.setattr(PretrainedConfig, "validate_rope", original_validate_rope) + _patch_hf_transformers_nested_rope_validation() + + nested_rope_parameters = { + "full_attention": {"rope_type": "default"}, + "original_max_position_embeddings": 32768, + } + PretrainedConfig.validate_rope( + SimpleNamespace(rope_parameters=nested_rope_parameters) + ) + assert nested_rope_parameters == {"full_attention": {"rope_type": "default"}} + + flat_rope_parameters = { + "rope_type": "linear", + "factor": 8.0, + "rope_theta": 500000.0, + } + PretrainedConfig.validate_rope( + SimpleNamespace(rope_parameters=flat_rope_parameters) + ) + assert flat_rope_parameters == { + "rope_type": "linear", + "factor": 8.0, + "rope_theta": 500000.0, + } + assert len(calls) == 2 + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 54f3086431..20d0d73afb 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -273,7 +273,7 @@ def _patch_hf_transformers_nested_rope_validation() -> None: layer_types = set(rope_parameters) & set( hf_configuration_utils.ALLOWED_LAYER_TYPES ) - if shared_keys := set(rope_parameters) - layer_types: + if layer_types and (shared_keys := set(rope_parameters) - layer_types): for key in shared_keys: del rope_parameters[key] logger.warning( From 2e50824766ead3e30f53cba711e8c7ffb2bb6b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20el=20Cheikh?= <157884230+melcheikh@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:27:54 -0300 Subject: [PATCH 023/143] [Bugfix] Fall back to native sampling when FlashInfer cannot target the GPU (#48956) Signed-off-by: martin el cheikh Co-authored-by: Claude --- tests/v1/sample/test_topk_topp_sampler.py | 38 ++++++++++++++++++++++ vllm/v1/sample/ops/topk_topp_sampler.py | 39 ++++++++++++++++++++++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/tests/v1/sample/test_topk_topp_sampler.py b/tests/v1/sample/test_topk_topp_sampler.py index 0b570f49d9..8ff16e5599 100644 --- a/tests/v1/sample/test_topk_topp_sampler.py +++ b/tests/v1/sample/test_topk_topp_sampler.py @@ -237,6 +237,44 @@ def test_flashinfer_sampler(): ) +@pytest.mark.skipif( + not FLASHINFER_TOPK_TOPP_SUPPORTED, + reason="Requires CUDA and flashinfer", +) +def test_flashinfer_sampler_fallback_when_jit_cannot_target_gpu(monkeypatch): + """FlashInfer swallows arch-detection errors (e.g. SM 12.x with a CUDA + toolkit older than 12.9), leaving an empty target-arch set that makes + every JIT spec fail with a misleading "requires sm75 or higher" error at + first use, killing the engine during startup profiling (issue #42393). + The sampler gate must detect this and fall back to native sampling, and + surface the real reason when the user explicitly opted in. + """ + import flashinfer.jit.core + from flashinfer.compilation_context import CompilationContext + + from vllm.v1.sample.ops.topk_topp_sampler import flashinfer_sampler_supported + + def fake_check_cuda_arch(): + raise RuntimeError("FlashInfer requires GPUs with sm75 or higher") + + def fake_normalize_cuda_arch(major, minor): + raise RuntimeError("SM 12.x requires CUDA >= 12.9") + + monkeypatch.setattr(flashinfer.jit.core, "check_cuda_arch", fake_check_cuda_arch) + monkeypatch.setattr( + CompilationContext, + "_normalize_cuda_arch", + staticmethod(fake_normalize_cuda_arch), + ) + + monkeypatch.delenv("VLLM_USE_FLASHINFER_SAMPLER", raising=False) + assert flashinfer_sampler_supported() is False + + monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "1") + with pytest.raises(RuntimeError, match="SM 12.x requires CUDA >= 12.9"): + flashinfer_sampler_supported() + + # ============================================================================= # Triton kernel tests # ============================================================================= diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index 06b3fc3423..477e99c2fa 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -10,6 +10,7 @@ from vllm._aiter_ops import rocm_aiter_ops from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode from vllm.logger import init_logger from vllm.platforms import CpuArchEnum, current_platform +from vllm.platforms.interface import DeviceCapability from vllm.triton_utils import HAS_TRITON if HAS_TRITON: @@ -41,12 +42,46 @@ def _skip_aiter_sampler_on_gfx1250() -> bool: return on_gfx1250() +def _flashinfer_jit_unsupported_reason(capability: DeviceCapability) -> str | None: + """Return why FlashInfer JIT codegen cannot target the current GPU, or + None if it can. + + FlashInfer swallows arch-detection errors when building its compilation + context (e.g. SM 12.x with a CUDA toolkit older than 12.9), leaving an + empty target-arch set that makes every JIT spec fail with a misleading + "requires sm75 or higher" error at first use — killing the engine during + startup profiling (https://github.com/vllm-project/vllm/issues/42393). + """ + try: + from flashinfer.jit.core import check_cuda_arch + except ImportError: + return None + try: + check_cuda_arch() + return None + except RuntimeError as e: + reason = str(e) + try: + # Re-derive the real error that FlashInfer swallowed during arch + # detection, e.g. "SM 12.x requires CUDA >= 12.9". + from flashinfer.compilation_context import CompilationContext + + CompilationContext._normalize_cuda_arch(capability.major, capability.minor) + except RuntimeError as e: + reason = str(e) + except Exception: + # FlashInfer internals changed; keep the original error message. + pass + return reason + + def flashinfer_sampler_supported() -> bool: """Decide whether FlashInfer's top-p/top-k sampler can be used. Returns False (with appropriate logging) when ``VLLM_USE_FLASHINFER_SAMPLER`` is 0, when the platform isn't CUDA, when the GPU's compute capability is - unsupported. Raises ``RuntimeError`` if the user explicitly opted in + unsupported, or when FlashInfer cannot JIT-compile for the current + GPU/CUDA toolchain. Raises ``RuntimeError`` if the user explicitly opted in via the env var but FlashInfer is unavailable. Assumes flashinfer is installed, as guaranteed by ``requirements/cuda.txt``; @@ -73,6 +108,8 @@ def flashinfer_sampler_supported() -> bool: unsupported_reason = ( f"unsupported compute capability {capability.as_version_str()}" ) + else: + unsupported_reason = _flashinfer_jit_unsupported_reason(capability) if unsupported_reason is None: logger.info_once("Using FlashInfer for top-p & top-k sampling.", scope="global") From f46968cb6e28ff89182bfa6fc291b4f851ab07d1 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 17 Sep 2026 05:42:59 -0700 Subject: [PATCH 024/143] [CI] Make ci-clean-log.sh portable to macOS/BSD sed (#57367) Signed-off-by: khluu --- .buildkite/scripts/ci-clean-log.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/ci-clean-log.sh b/.buildkite/scripts/ci-clean-log.sh index e2e21483d5..2fe45f3b77 100644 --- a/.buildkite/scripts/ci-clean-log.sh +++ b/.buildkite/scripts/ci-clean-log.sh @@ -10,11 +10,20 @@ fi INPUT_FILE="$1" +# GNU sed uses "-i [-r]" while BSD/macOS sed needs "-i '' [-E]"; detect once. +if sed --version >/dev/null 2>&1; then + sedi() { sed -i "$@"; } + SED_ERE=-r +else + sedi() { sed -i '' "$@"; } + SED_ERE=-E +fi + # Strip timestamps -sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE" +sedi 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE" # Strip Buildkite inline timestamp markers (ESC _bk;t= BEL) -sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE" +sedi 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE" # Strip colorization -sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE" +sedi "$SED_ERE" 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE" From 8be3ca35f311fc205262122aedcf775d56ebc3ac Mon Sep 17 00:00:00 2001 From: Matt <156021403+mawong-amd@users.noreply.github.com> Date: Thu, 17 Sep 2026 05:52:02 -0700 Subject: [PATCH 025/143] [ROCm][CI] Fix AMD CI pipeline upload rejected by an invalid block-step key (#57375) Signed-off-by: Matthew Wong Co-authored-by: Claude Opus 5 (1M context) --- .buildkite/test-amd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index d038d66176..2f5a84aaf1 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2414,7 +2414,7 @@ steps: - pytest -v -s entrypoints/serve/instrumentator/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" -- label: ":amd: (MI300) Rust Frontend Serve/Admin Coverage" +- label: ":amd: (MI300) Rust Frontend Serve-Admin Coverage" timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false From 0ef8366ba39fc38b88c60b17ee2ddf05aa8a7596 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:53:11 +0100 Subject: [PATCH 026/143] [Frontend] Only show the summary line of config docstrings in `--help` (#57357) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) --- tests/utils_/test_argparse_utils.py | 17 +++++++++++++++++ vllm/utils/argparse_utils.py | 25 ++++++++++++++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/tests/utils_/test_argparse_utils.py b/tests/utils_/test_argparse_utils.py index baf57d994f..89e0f9977d 100644 --- a/tests/utils_/test_argparse_utils.py +++ b/tests/utils_/test_argparse_utils.py @@ -524,3 +524,20 @@ def test_flat_product(): (3, 4, "a", 5, 6), (3, 4, "b", 5, 6), ] + + +def test_group_description_is_summary_only(): + """Group descriptions are config docstrings, too long for the terminal.""" + parser = FlexibleArgumentParser() + group = parser.add_argument_group( + title="MyConfig", + description="Summary line.\n\nDetails which only belong in the docs.", + ) + group.add_argument("--my-arg") + + assert "Summary line." in parser.format_help() + assert "only belong in the docs" not in parser.format_help() + + parser._search_keyword = "myconfig" + assert "Summary line." in parser.format_help() + assert "only belong in the docs" not in parser.format_help() diff --git a/vllm/utils/argparse_utils.py b/vllm/utils/argparse_utils.py index e573cbc293..d214da3402 100644 --- a/vllm/utils/argparse_utils.py +++ b/vllm/utils/argparse_utils.py @@ -93,6 +93,11 @@ def human_readable_int_or_auto(value: str) -> int: return human_readable_int(value) +def summary_line(description: str | None) -> str: + """Return the first paragraph of a config docstring as a single line.""" + return " ".join((description or "").split("\n\n", 1)[0].split()) + + class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter): """SortedHelpFormatter that sorts arguments by their option strings.""" @@ -203,7 +208,7 @@ class FlexibleArgumentParser(ArgumentParser): for group in self._action_groups: if group.title and group.title.lower() == search_keyword: formatter.start_section(group.title) - formatter.add_text(group.description) + formatter.add_text(summary_line(group.description)) formatter.add_arguments(group._group_actions) formatter.end_section() formatter.add_text(self._json_tip) @@ -242,12 +247,18 @@ class FlexibleArgumentParser(ArgumentParser): # positionals, optionals and user-defined groups formatter.start_section("Config Groups") config_groups = "" - for group in self._action_groups: - if not group._group_actions: - continue - title = group.title - description = group.description or "" - config_groups += f"{title: <24}{description}\n" + groups = [g for g in self._action_groups if g._group_actions] + titles = [g.title or "" for g in groups] + title_width = max((len(title) for title in titles), default=0) + 1 + width = max(formatter._width - formatter._current_indent, title_width + 20) + for title, group in zip(titles, groups): + lines = textwrap.wrap( + summary_line(group.description), + width, + initial_indent=f"{title: <{title_width}}", + subsequent_indent=" " * title_width, + ) + config_groups += ("\n".join(lines) or title) + "\n" formatter.add_text(config_groups) formatter.end_section() From 75c71390d5b399f5397a9166920fc45902f99f14 Mon Sep 17 00:00:00 2001 From: CaiJohn Date: Thu, 17 Sep 2026 20:57:47 +0800 Subject: [PATCH 027/143] [Bugfix] Escape control characters in xgrammar choice grammar (#48115) Signed-off-by: john <6712432+CaiJohn@users.noreply.github.com> Co-authored-by: john <6712432+CaiJohn@users.noreply.github.com> --- tests/v1/structured_output/test_utils.py | 46 ++++++++++++++++++++++++ vllm/v1/structured_output/utils.py | 15 ++++++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/tests/v1/structured_output/test_utils.py b/tests/v1/structured_output/test_utils.py index f5f8020220..3e4a12f7e5 100644 --- a/tests/v1/structured_output/test_utils.py +++ b/tests/v1/structured_output/test_utils.py @@ -5,9 +5,12 @@ import pytest from xgrammar import Grammar from xgrammar.testing import _is_grammar_accept_string +from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.structured_output.backend_xgrammar import ( has_xgrammar_unsupported_json_features, + validate_xgrammar_grammar, ) +from vllm.v1.structured_output.utils import choice_as_grammar pytestmark = pytest.mark.cpu_test @@ -232,3 +235,46 @@ def test_pattern_properties_constrains_keys_and_values(pattern_properties_schema assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}') assert not grammar_accepts(pattern_properties_schema, '{"other": 5}') assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}') + + +class TestIsGrammarAcceptString: + class TestPR48115Regressions: + @pytest.mark.parametrize("codepoint", [*range(0x20), 0x7F]) + def test_choice_as_grammar_preserves_control_characters(self, codepoint): + choice = f"a{chr(codepoint)}f" + grammar = Grammar.from_ebnf(choice_as_grammar([choice])) + + assert _is_grammar_accept_string(grammar, choice) + assert not _is_grammar_accept_string(grammar, "a") + assert not _is_grammar_accept_string(grammar, "af") + assert not _is_grammar_accept_string(grammar, f"a\\u{codepoint:04x}f") + + @pytest.mark.parametrize( + ("choice", "other"), + [ + (r"a\nf", "a\nf"), + ('a quote " and a backslash \\', 'a quote " and a backslash '), + ("café", "cafe"), + ("日本語", "日本"), + ("😀", "😁"), + ], + ) + def test_choice_as_grammar_preserves_literal_choices(self, choice, other): + grammar = Grammar.from_ebnf(choice_as_grammar(["yes", choice])) + + assert _is_grammar_accept_string(grammar, "yes") + assert _is_grammar_accept_string(grammar, choice) + assert not _is_grammar_accept_string(grammar, other) + assert not _is_grammar_accept_string(grammar, choice + "extra") + + def test_validate_xgrammar_preserves_multiline_choice(self): + structured_outputs = StructuredOutputsParams(choice=["yes", "no\nplease"]) + validate_xgrammar_grammar( + SamplingParams(structured_outputs=structured_outputs) + ) + + assert structured_outputs.choice is None + grammar = Grammar.from_ebnf(structured_outputs.grammar) + assert _is_grammar_accept_string(grammar, "yes") + assert _is_grammar_accept_string(grammar, "no\nplease") + assert not _is_grammar_accept_string(grammar, r"no\nplease") diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 4969789434..4bb438d6f3 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -567,9 +567,18 @@ def convert_lark_to_ebnf(grammar_str: str) -> str: def choice_as_grammar(choice: list[str]) -> str: def escape_ebnf_string(s: str) -> str: - """Escape special characters in a EBNF string.""" - # Escape double quotes and backslashes - return re.sub(r'(["\\])', r"\\\1", s) + """Escape EBNF literals, including raw LF, CR, and NUL terminators.""" + escapes = {"\\": r"\\", '"': r"\"", "\n": r"\n", "\r": r"\r", "\t": r"\t"} + + def escape_char(ch: str) -> str: + if ch in escapes: + return escapes[ch] + # Escape remaining C0 controls (U+0000-U+001F) and DEL (U+007F). + if ord(ch) < 0x20 or ord(ch) == 0x7F: + return f"\\u{ord(ch):04x}" + return ch + + return "".join(escape_char(ch) for ch in s) escaped_choices = (escape_ebnf_string(c) for c in choice) grammar = "root ::= " + " | ".join(f'"{c}"' for c in escaped_choices) From fe284ea17d606ceba4a5d3d55d3e3d9a34f478d9 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 17 Sep 2026 09:35:34 -0400 Subject: [PATCH 028/143] [BugFxi] Fix DeepGEMM FP8 workspace over allocation (#53914) Signed-off-by: Lucas Wilkinson Signed-off-by: Lucas Wilkinson Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- tests/kernels/moe/test_deepgemm.py | 40 +++++++++++++++++++ .../layers/fused_moe/deep_gemm_utils.py | 3 +- .../layers/fused_moe/experts/deep_gemm_moe.py | 33 ++++++++++++++- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/tests/kernels/moe/test_deepgemm.py b/tests/kernels/moe/test_deepgemm.py index f6df64c51b..cee0faa592 100644 --- a/tests/kernels/moe/test_deepgemm.py +++ b/tests/kernels/moe/test_deepgemm.py @@ -26,6 +26,10 @@ from vllm.model_executor.layers.fused_moe.config import ( ) from vllm.model_executor.layers.fused_moe.deep_gemm_utils import ( deepgemm_moe_permute, + ep_gather, +) +from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( + _fp8_workspace_shape, ) from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import ( TritonOrDeepGemmExperts, @@ -43,6 +47,42 @@ from vllm.utils.deep_gemm import ( BLOCK_SIZE = [128, 128] +def test_ep_gather_uses_64_bit_row_offsets(): + hidden_size = 4096 + source_row = 1 << 18 + input_tensor = torch.empty( + (source_row + 1, hidden_size), + device="cuda", + dtype=torch.bfloat16, + ) + input_tensor[source_row].fill_(1) + output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16) + + ep_gather( + input_tensor=input_tensor, + recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64), + recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32), + input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32), + expert_map=None, + output_tensor=output, + ) + + torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0) + + +@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("num_columns", [2048, 6144, 6145]) +def test_fp8_workspace_shape(workspace_dtype, num_columns): + num_rows = 17 + shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype) + + allocated_bytes = math.prod(shape) * workspace_dtype.itemsize + required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize + + assert allocated_bytes >= required_bytes + assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize + + @pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init): hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16) diff --git a/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py b/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py index 4ea864eaac..33a2296955 100644 --- a/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py +++ b/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py @@ -393,12 +393,13 @@ def _fwd_kernel_ep_gather( source_token_index = tl.load( input_index + cur_token * input_index_stride0 + topk_index ) + source_token_index_i64 = source_token_index.to(tl.int64) acc_weight = tl.load( recv_topk_weight + cur_token * recv_topk_weight_stride0 + topk_index ) tmp = tl.load( input_tensor - + source_token_index * input_tensor_stride0 + + source_token_index_i64 * input_tensor_stride0 + cur_block * BLOCK_D + off_d ) diff --git a/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py index fe95b36858..699b029bad 100644 --- a/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py @@ -50,6 +50,21 @@ from vllm.utils.import_utils import has_deep_gemm logger = init_logger(__name__) +def _fp8_workspace_shape( + num_rows: int, num_columns: int, workspace_dtype: torch.dtype +) -> tuple[int, int]: + """Size an FP8 byte buffer stored in a workspace of another dtype.""" + bytes_per_workspace_element = workspace_dtype.itemsize + fp8_columns_per_workspace_element = ( + bytes_per_workspace_element // torch.float8_e4m3fn.itemsize + ) + assert bytes_per_workspace_element % torch.float8_e4m3fn.itemsize == 0 + return ( + num_rows, + -(-num_columns // fp8_columns_per_workspace_element), + ) + + def _valid_deep_gemm_shape(M: int, N: int, K: int) -> bool: align = get_mk_alignment_for_contiguous_layout()[0] return align <= M and N % align == 0 and K % align == 0 @@ -216,7 +231,14 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): assert M_sum % align_used == 0 activation_out_dim = self.adjust_N_for_activation(N, activation) - workspace1 = (M_sum, max(activation_out_dim, K)) + # workspace1 is allocated in the activation dtype by the workspace + # manager, but is only ever viewed and used as FP8 in apply(). Size it + # by bytes instead of reserving one BF16/FP16 element per FP8 element. + workspace1 = _fp8_workspace_shape( + M_sum, + max(activation_out_dim, K), + self.workspace_dtype(self.moe_config.in_dtype), + ) workspace2 = (M_sum, max(N, K)) output = (M, K) return (workspace1, workspace2, output) @@ -480,7 +502,14 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular): assert M_sum % align_used == 0 activation_out_dim = self.adjust_N_for_activation(N, activation) - workspace1 = (M_sum, max(activation_out_dim, K)) + # workspace1 holds the permuted and requantized FP8 activations. The + # workspace manager allocates it in the model activation dtype, so + # account for the dtype sizes rather than overallocating BF16/FP16. + workspace1 = _fp8_workspace_shape( + M_sum, + max(activation_out_dim, K), + self.workspace_dtype(self.moe_config.in_dtype), + ) workspace2 = (M_sum, max(N, K)) output = (M, K) return (workspace1, workspace2, output) From 0eae9acd4d01574e12d4ecf6a0229813f7fdb799 Mon Sep 17 00:00:00 2001 From: Shantipriya Parida Date: Thu, 17 Sep 2026 16:39:26 +0300 Subject: [PATCH 029/143] [Bugfix][ROCm][MoE] Fall back instead of crashing when AITER MoE is requested for a non-gated (is_act_and_mul=False) model (#56590) Signed-off-by: Shantipriya Parida --- .../moe/test_unquantized_backend_selection.py | 60 +++++++++++++++++++ .../layers/fused_moe/oracle/unquantized.py | 28 ++++++++- 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/tests/kernels/moe/test_unquantized_backend_selection.py b/tests/kernels/moe/test_unquantized_backend_selection.py index d4c037d978..9d0f4d6900 100644 --- a/tests/kernels/moe/test_unquantized_backend_selection.py +++ b/tests/kernels/moe/test_unquantized_backend_selection.py @@ -6,6 +6,7 @@ import pytest import torch from tests.kernels.moe.utils import make_dummy_moe_config +from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import RoutingMethodType from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( UnquantizedMoeBackend, @@ -228,6 +229,65 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer): assert expert_cls is not None +@patch( + "vllm.utils.flashinfer.has_flashinfer", + return_value=False, +) +@patch( + "vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops." + "is_fused_moe_enabled", + return_value=True, +) +@patch( + "vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops." + "is_rdna_aiter_enabled", + return_value=False, +) +@pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific backend selection test" +) +def test_select_rocm_aiter_backend_non_gated_activation_falls_back( + mock_rdna_disabled, mock_aiter_enabled, mock_has_flashinfer, monkeypatch +): + """Test ROCm backend selection falls back (not raises) for non-gated MoE.""" + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1") + + with patch( + "vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform" + ) as mock_platform: + mock_platform.is_cuda.return_value = False + mock_platform.is_rocm.return_value = True + mock_platform.is_cpu.return_value = False + mock_platform.is_xpu.return_value = False + mock_platform.is_tpu.return_value = False + mock_platform.is_out_of_tree.return_value = False + + moe_config = make_dummy_moe_config(activation=MoEActivation.SILU_NO_MUL) + assert moe_config.is_act_and_mul is False + + selected_backend, expert_cls = select_unquantized_moe_backend( + moe_config=moe_config, + ) + + assert selected_backend != UnquantizedMoeBackend.AITER + assert expert_cls is not None + + +@pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific backend selection test" +) +def test_explicit_aiter_backend_non_gated_activation_still_raises(): + """Explicit `--moe-backend aiter` still raises for non-gated MoE; + only the env-var opt-in path falls back.""" + moe_config = make_dummy_moe_config(activation=MoEActivation.SILU_NO_MUL) + moe_config.moe_backend = "aiter" # explicit pin, not "auto" + assert moe_config.is_act_and_mul is False + + with pytest.raises(ValueError): + select_unquantized_moe_backend(moe_config=moe_config) + + @patch( "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config", return_value=(True, None), diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index 9101787059..d1437933c3 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -305,7 +305,33 @@ def select_unquantized_moe_backend( AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER) else: backend = UnquantizedMoeBackend.AITER - return _return_or_raise(backend, moe_config, activation_format) + reason = None + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, moe_config, None, None, activation_format + ) + if supported: + logger.info_once(_make_log_backend(backend)) + return backend, k_cls + # AITER was explicitly requested but does not support this + # deployment configuration (e.g. a non-gated MoE activation, + # which AiterExperts._supports_no_act_and_mul() rejects + # unconditionally). Rather than hard-crashing at engine init, + # warn — explicitly stating that this is falling back, and + # which backends will be tried next — and fall back to the + # remaining backends in priority order below. + logger.warning_once( + "VLLM_ROCM_USE_AITER_MOE=1 was requested, but %s " + "Falling back to try the remaining available MoE backends: %s.", + _make_log_unsupported(backend, reason), + ", ".join( + b.value + for b in AVAILABLE_BACKENDS + if b != UnquantizedMoeBackend.AITER + ), + ) + if UnquantizedMoeBackend.AITER in AVAILABLE_BACKENDS: + AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER) for backend in AVAILABLE_BACKENDS: for k_cls in backend_to_kernel_cls(backend): From 117cf43fb65713a0a7a3c35b23be078d5fb319d4 Mon Sep 17 00:00:00 2001 From: Rukhaiya2004 <162583766+Rukhaiya2004@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:15:13 +0530 Subject: [PATCH 030/143] [HARDWARE][POWER] Enable W8A8 INT8 MoE on POWER (#55316) Signed-off-by: Rukhaiya Signed-off-by: Akash kaothalkar Co-authored-by: Akash kaothalkar Co-authored-by: Antigravity Co-authored-by: Li, Jiang --- cmake/cpu_extension.cmake | 6 + csrc/cpu/cpu_fused_moe_int8.cpp | 19 +- csrc/cpu/cpu_types_vsx.hpp | 31 +- .../micro_gemm/cpu_micro_gemm_int8_vsx.hpp | 287 ++++++++++++++++++ csrc/cpu/torch_bindings.cpp | 9 +- tests/kernels/moe/test_cpu_fused_moe.py | 22 +- .../layers/fused_moe/experts/cpu_moe.py | 177 +++++++++++ .../layers/fused_moe/oracle/int8.py | 8 +- 8 files changed, 537 insertions(+), 22 deletions(-) create mode 100644 csrc/cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index ff2911dc11..6359a96887 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -522,6 +522,12 @@ if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) ${VLLM_EXT_SRC}) endif() +if (POWER10_FOUND OR POWER11_FOUND) + set(VLLM_EXT_SRC + "csrc/cpu/cpu_fused_moe_int8.cpp" + ${VLLM_EXT_SRC}) +endif() + if(USE_ONEDNN) set(VLLM_EXT_SRC "csrc/cpu/dnnl_kernels.cpp" diff --git a/csrc/cpu/cpu_fused_moe_int8.cpp b/csrc/cpu/cpu_fused_moe_int8.cpp index 649317b478..27d0f6a7d5 100644 --- a/csrc/cpu/cpu_fused_moe_int8.cpp +++ b/csrc/cpu/cpu_fused_moe_int8.cpp @@ -26,10 +26,23 @@ #define NEON_DISPATCH(SCALAR_TYPE, ...) case cpu_utils::ISA::NEON: #endif +#if defined(__powerpc64__) + #include "cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp" + #define VSX_DISPATCH(SCALAR_TYPE, ...) \ + case cpu_utils::ISA::VSX: { \ + using gemm_t = \ + cpu_micro_gemm::MicroGemmINT8; \ + return __VA_ARGS__(); \ + } +#else + #define VSX_DISPATCH(SCALAR_TYPE, ...) case cpu_utils::ISA::VSX: +#endif + #define CPU_INT8_ISA_DISPATCH_IMPL(ISA_TYPE, SCALAR_TYPE, ...) \ [&] { \ switch (ISA_TYPE) { \ NEON_DISPATCH(SCALAR_TYPE, __VA_ARGS__) \ + VSX_DISPATCH(SCALAR_TYPE, __VA_ARGS__) \ default: { \ TORCH_CHECK(false, "Invalid CPU ISA type."); \ } \ @@ -81,9 +94,9 @@ void fused_moe_int8_impl( const int32_t thread_num = cpu_utils::get_max_threads(); const int32_t w13_input_buffer_size = cpu_utils::round_up<64>( - gemm_m_tile_size * input_size_13 * sizeof(int8_t)); - const int32_t w2_input_buffer_size = - cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 * sizeof(int8_t)); + (gemm_m_tile_size * input_size_13 + 64) * sizeof(int8_t)); + const int32_t w2_input_buffer_size = cpu_utils::round_up<64>( + (gemm_m_tile_size * input_size_2 + 64) * sizeof(int8_t)); const int32_t w13_n_tile_size = [&]() { const int64_t cache_size = cpu_utils::get_available_l2_size(); diff --git a/csrc/cpu/cpu_types_vsx.hpp b/csrc/cpu/cpu_types_vsx.hpp index d46906130e..eaea921d98 100644 --- a/csrc/cpu/cpu_types_vsx.hpp +++ b/csrc/cpu/cpu_types_vsx.hpp @@ -13,10 +13,10 @@ namespace vec_op { struct fp8_e4m3_tag {}; struct fp8_e5m2_tag {}; -#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \ - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ - AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \ - AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) +// Note: FP16 (Half) is not supported on POWER VSX — no FP16Vec16 type. #define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__)) @@ -416,6 +416,7 @@ struct FP32Vec8 : public Vec { tmp.val[0] = two_x; tmp.val[1] = two_x; FP32Vec8 temp_vec(tmp); + FP32Vec8 exp_vec = temp_vec.exp(); vector float e = temp_vec.exp().reg.val[0]; vector float num = vec_sub(e, one); @@ -634,6 +635,16 @@ struct FP32Vec16 : public Vec { reg.val[3] = vec_ctf(v.reg.val[3], 0); } + explicit FP32Vec16(const float* __restrict__ base, const INT32Vec16& index) { + INT32Vec16::AliasReg idx; + idx.reg = index.reg; + AliasReg ar; + for (int i = 0; i < VEC_ELEM_NUM; ++i) { + ar.values[i] = base[idx.values[i]]; + } + reg = ar.reg; + } + FP32Vec16 operator*(const FP32Vec16& b) const { return FP32Vec16(f32x4x4_t({vec_mul(reg.val[0], b.reg.val[0]), vec_mul(reg.val[1], b.reg.val[1]), @@ -648,6 +659,13 @@ struct FP32Vec16 : public Vec { vec_add(reg.val[3], b.reg.val[3])})); } + FP32Vec16 operator-() const { + const __vector float zero = vec_splats(0.0f); + return FP32Vec16( + f32x4x4_t({vec_sub(zero, reg.val[0]), vec_sub(zero, reg.val[1]), + vec_sub(zero, reg.val[2]), vec_sub(zero, reg.val[3])})); + } + FP32Vec16 operator-(const FP32Vec16& b) const { return FP32Vec16(f32x4x4_t({vec_sub(reg.val[0], b.reg.val[0]), vec_sub(reg.val[1], b.reg.val[1]), @@ -655,11 +673,6 @@ struct FP32Vec16 : public Vec { vec_sub(reg.val[3], b.reg.val[3])})); } - FP32Vec16 operator-() const { - return FP32Vec16( - f32x4x4_t({-reg.val[0], -reg.val[1], -reg.val[2], -reg.val[3]})); - } - FP32Vec16 operator/(const FP32Vec16& b) const { return FP32Vec16(f32x4x4_t({vec_div(reg.val[0], b.reg.val[0]), vec_div(reg.val[1], b.reg.val[1]), diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp new file mode 100644 index 0000000000..6033353728 --- /dev/null +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp @@ -0,0 +1,287 @@ +#ifndef CPU_MICRO_GEMM_INT8_VSX_HPP +#define CPU_MICRO_GEMM_INT8_VSX_HPP + +#include +#include +#include +#include +#include + +#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp" + +#include +#include +#include + +namespace cpu_micro_gemm { + +namespace vsx_int8 { + +constexpr int32_t K = 4; +constexpr int32_t Mr = 4; +constexpr int32_t Nr = 32; + +// --------------------------------------------------------------------------- +// 8-Accumulator MMA 4x32 micro-GEMM (POWER10 xvi8ger4pp). +// +// Computes 4 rows x 32 columns (128 outputs) using all 8 hardware accumulators. +// Operates on pre-biased uint8 weights (B + 128) and subtracts 128 * a_sums. +// Works for all m in [1, 4] with zero scalar loop overhead. +// --------------------------------------------------------------------------- +__attribute__((target("cpu=power10"))) FORCE_INLINE void gemm_mma_4x32( + const int8_t* __restrict__ a_packed, const uint8_t* __restrict__ b_panel, + const int32_t* __restrict__ a_sums, int32_t* __restrict__ c_ptr, + const int32_t m, const int32_t k_size, const int64_t ldc) { + TORCH_CHECK(m > 0 && m <= Mr); + TORCH_CHECK(k_size % K == 0); + + __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7; + __builtin_mma_xxsetaccz(&acc0); + __builtin_mma_xxsetaccz(&acc1); + __builtin_mma_xxsetaccz(&acc2); + __builtin_mma_xxsetaccz(&acc3); + __builtin_mma_xxsetaccz(&acc4); + __builtin_mma_xxsetaccz(&acc5); + __builtin_mma_xxsetaccz(&acc6); + __builtin_mma_xxsetaccz(&acc7); + + const int8_t* __restrict__ ap = a_packed; + const uint8_t* __restrict__ bp = b_panel; + +#pragma GCC unroll 4 + for (int32_t k = 0; k < k_size; k += 4) { + const __vector unsigned char avec = + vec_xl(0, reinterpret_cast(ap)); + __builtin_mma_xvi8ger4pp(&acc0, avec, vec_xl(0, bp)); + __builtin_mma_xvi8ger4pp(&acc1, avec, vec_xl(16, bp)); + __builtin_mma_xvi8ger4pp(&acc2, avec, vec_xl(32, bp)); + __builtin_mma_xvi8ger4pp(&acc3, avec, vec_xl(48, bp)); + __builtin_mma_xvi8ger4pp(&acc4, avec, vec_xl(64, bp)); + __builtin_mma_xvi8ger4pp(&acc5, avec, vec_xl(80, bp)); + __builtin_mma_xvi8ger4pp(&acc6, avec, vec_xl(96, bp)); + __builtin_mma_xvi8ger4pp(&acc7, avec, vec_xl(112, bp)); + ap += 16; + bp += 128; + } + + __vector signed int res[4]; +#define STORE_ACC(ACC, CG_IDX) \ + { \ + __builtin_mma_disassemble_acc(res, &ACC); \ + for (int r = 0; r < m; ++r) { \ + const __vector signed int vcorr = vec_splats(128 * a_sums[r]); \ + const __vector signed int val = vec_sub(res[r], vcorr); \ + vec_xst(val, 0, \ + reinterpret_cast(c_ptr + r * ldc + (CG_IDX) * 4)); \ + } \ + } + STORE_ACC(acc0, 0); + STORE_ACC(acc1, 1); + STORE_ACC(acc2, 2); + STORE_ACC(acc3, 3); + STORE_ACC(acc4, 4); + STORE_ACC(acc5, 5); + STORE_ACC(acc6, 6); + STORE_ACC(acc7, 7); +#undef STORE_ACC +} + +} // namespace vsx_int8 + +// ============================================================================= +// MicroGemmINT8 specialisation for VSX (POWER10+) +// ============================================================================= +template +class MicroGemmINT8 { + public: + static constexpr int32_t K = vsx_int8::K; // 4 + static constexpr int32_t Mr = vsx_int8::Mr; // 4 + static constexpr int32_t Nr = vsx_int8::Nr; // 32 + static constexpr int32_t NrGemv = Nr; + static constexpr int32_t MaxMSize = 8; + static constexpr int32_t NSize = Nr; + static constexpr int32_t WeightOCGroupSize = Nr; + static_assert(MaxMSize % Mr == 0); + + // ------------------------------------------------------------------------- + // quantize_row: scalar_t -> INT8 + per-token scale + // Single-pass 16-element vector conversion with aligned 16-byte stores. + // ------------------------------------------------------------------------- + static FORCE_INLINE void quantize_row(const scalar_t* input, int8_t* output, + float& scale, const int32_t size) { + TORCH_CHECK_EQ(size % 16, 0); + + const __vector signed short z = vec_splats((signed short)0); + + auto load_lo_hi = + [&](int32_t i) -> std::pair<__vector float, __vector float> { + if constexpr (std::is_same_v) { + const float* p = reinterpret_cast(input) + i; + return {vec_xl(0, p), vec_xl(16, p)}; + } else { + const signed short* p = + reinterpret_cast(input + i); + const __vector signed short raw = (__vector signed short)vec_xl(0, p); + return {(__vector float)vec_mergeh(z, raw), + (__vector float)vec_mergel(z, raw)}; + } + }; + + __vector float max_v = vec_splats(0.0f); + for (int32_t i = 0; i < size; i += 8) { + auto [lo, hi] = load_lo_hi(i); + max_v = vec_max(max_v, vec_abs(lo)); + max_v = vec_max(max_v, vec_abs(hi)); + } + __vector float t = vec_max(max_v, vec_sld(max_v, max_v, 8)); + t = vec_max(t, vec_sld(t, t, 4)); + const float abs_max = std::max(vec_extract(t, 0), 1.0e-7f); + scale = abs_max / 127.0f; + const __vector float inv_scale = vec_splats(127.0f / abs_max); + + for (int32_t i = 0; i < size; i += 16) { + auto [lo0, hi0] = load_lo_hi(i); + auto [lo1, hi1] = load_lo_hi(i + 8); + + const __vector signed int i0 = + vec_cts(vec_round(vec_mul(lo0, inv_scale)), 0); + const __vector signed int i1 = + vec_cts(vec_round(vec_mul(hi0, inv_scale)), 0); + const __vector signed int i2 = + vec_cts(vec_round(vec_mul(lo1, inv_scale)), 0); + const __vector signed int i3 = + vec_cts(vec_round(vec_mul(hi1, inv_scale)), 0); + + const __vector signed short s0 = vec_packs(i0, i1); + const __vector signed short s1 = vec_packs(i2, i3); + const __vector signed char s8 = vec_packs(s0, s1); + + vec_xst(s8, 0, reinterpret_cast(output + i)); + } + } + + // ------------------------------------------------------------------------- + // dequantize_tile: INT32 accumulator -> FP32 scaled output + // ------------------------------------------------------------------------- + static FORCE_INLINE void dequantize_tile( + int32_t* input, float* output, const float* __restrict__ input_scales, + const float* __restrict__ weight_scales, const int32_t m, const int32_t n, + const int32_t stride) { + TORCH_CHECK_EQ(n % 4, 0); + for (int32_t mi = 0; mi < m; ++mi) { + const __vector float a_scale = vec_splats(input_scales[mi]); + for (int32_t ni = 0; ni < n; ni += 4) { + const __vector signed int i32 = vec_xl(0, input + mi * stride + ni); + const __vector float wscale = vec_xl(0, weight_scales + ni); + vec_xst(vec_mul(vec_ctf(i32, 0), vec_mul(a_scale, wscale)), 0, + output + mi * stride + ni); + } + } + } + + // ------------------------------------------------------------------------- + // pack_input_from_rows: pack rows into 4-row MMA tiles and compute row sums + // Header: 64 bytes (contains int32 a_sums[0..7]) + // Data: a_packed + 64 (contains 4x4 tiles of A) + // ------------------------------------------------------------------------- + static void pack_input_from_rows(const int8_t* const* __restrict__ rows, + int8_t* __restrict__ a_packed, + const int32_t m, const int32_t k) { + TORCH_CHECK(m > 0 && m <= MaxMSize); + TORCH_CHECK_EQ(k % K, 0); + + int32_t* a_sums = reinterpret_cast(a_packed); + for (int32_t r = 0; r < 8; ++r) a_sums[r] = 0; + + int8_t* a_data = a_packed + 64; + + for (int32_t row_base = 0; row_base < m; row_base += Mr) { + const int32_t panel_m = std::min(Mr, m - row_base); + const int8_t* const* panel_rows = rows + row_base; + int32_t* panel_sums = a_sums + row_base; + int8_t* dst_panel = a_data + (row_base / Mr) * (k * Mr); + + for (int32_t ki = 0; ki < k; ki += 4) { + int8_t* dst = dst_panel + (ki / 4) * 16; + for (int32_t r = 0; r < 4; ++r) { + if (r < panel_m) { + const int8_t* src = panel_rows[r] + ki; + dst[r * 4 + 0] = src[0]; + dst[r * 4 + 1] = src[1]; + dst[r * 4 + 2] = src[2]; + dst[r * 4 + 3] = src[3]; + panel_sums[r] += + static_cast(src[0]) + src[1] + src[2] + src[3]; + } else { + dst[r * 4 + 0] = 0; + dst[r * 4 + 1] = 0; + dst[r * 4 + 2] = 0; + dst[r * 4 + 3] = 0; + } + } + } + } + } + + // ------------------------------------------------------------------------- + // pack_weight: [output_size, input_size] INT8 -> packed layout + // Pre-biases with +128 in [N/32][K/4][8 col-groups][4 cols x 4 k] layout + // ------------------------------------------------------------------------- + static void pack_weight(const int8_t* __restrict__ weight, + int8_t* __restrict__ packed_weight, + const int32_t output_size, const int32_t input_size) { + TORCH_CHECK_EQ(output_size % NSize, 0); + TORCH_CHECK_EQ(input_size % K, 0); + + uint8_t* dst_base = reinterpret_cast(packed_weight); + + for (int32_t o = 0; o < output_size; o += Nr) { + uint8_t* dst_panel = dst_base + o * input_size; + for (int32_t k = 0; k < input_size; k += 4) { + for (int32_t cg = 0; cg < 8; ++cg) { + uint8_t* dst = dst_panel + (k / 4) * 128 + cg * 16; + for (int32_t col = 0; col < 4; ++col) { + for (int32_t ki = 0; ki < 4; ++ki) { + const int8_t w = weight[(o + cg * 4 + col) * input_size + k + ki]; + dst[col * 4 + ki] = + static_cast(static_cast(w) + 128); + } + } + } + } + } + } + + // ------------------------------------------------------------------------- + // gemm: 8-accumulator MMA GEMM over 32-column tiles + // ------------------------------------------------------------------------- + void gemm(const int8_t* __restrict__ a_packed, + const int8_t* __restrict__ b_packed, int32_t* __restrict__ c, + const int32_t m, const int32_t k, const int64_t b_n_group_stride, + const int64_t ldc) const { + TORCH_CHECK(m > 0 && m <= MaxMSize); + TORCH_CHECK_EQ(k % K, 0); + (void)b_n_group_stride; + + const int32_t* a_sums = reinterpret_cast(a_packed); + const int8_t* a_data = a_packed + 64; + const uint8_t* b_u8 = reinterpret_cast(b_packed); + + for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr) { + const uint8_t* b_panel = b_u8 + n_idx * k; + + for (int32_t row_base = 0; row_base < m; row_base += Mr) { + const int32_t panel_m = std::min(Mr, m - row_base); + const int8_t* a_panel = a_data + (row_base / Mr) * (k * Mr); + const int32_t* panel_sums = a_sums + row_base; + + vsx_int8::gemm_mma_4x32(a_panel, b_panel, panel_sums, + c + row_base * ldc + n_idx, panel_m, k, ldc); + } + } + } +}; + +} // namespace cpu_micro_gemm + +#endif // CPU_MICRO_GEMM_INT8_VSX_HPP diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index 8f663a93c3..88170068de 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -883,8 +883,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "bool skip_weighted, " "str act, str isa) -> ()"); ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe); -#if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \ - !defined(__APPLE__) + +#if (defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \ + !defined(__APPLE__)) || \ + defined(__powerpc64__) ops.def( "prepack_moe_weight_int8(Tensor weight, Tensor(a1!) packed_weight, " "str isa) -> ()"); @@ -895,8 +897,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "Tensor? w2_bias, Tensor topk_weights, Tensor topk_id, bool " "skip_weighted, str act, str isa) -> ()"); ops.impl("cpu_fused_moe_int8", torch::kCPU, &cpu_fused_moe_int8); -#endif // #if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && - // !defined(__APPLE__) +#endif // ARM_I8MM+BF16 or __powerpc64__ ops.def( "mla_decode_kvcache(" " Tensor! out, Tensor query, Tensor kv_cache," diff --git a/tests/kernels/moe/test_cpu_fused_moe.py b/tests/kernels/moe/test_cpu_fused_moe.py index cf83ef6fc0..d4c5437221 100644 --- a/tests/kernels/moe/test_cpu_fused_moe.py +++ b/tests/kernels/moe/test_cpu_fused_moe.py @@ -306,17 +306,28 @@ def test_cpu_fused_moe( @pytest.mark.skipif( - current_platform.get_cpu_architecture() != CpuArchEnum.ARM, - reason="Requires Arm CPU", + current_platform.get_cpu_architecture() + not in (CpuArchEnum.ARM, CpuArchEnum.POWERPC), + reason="Requires Arm or POWER CPU", ) @pytest.mark.parametrize("batch_size", BATCH_SIZE) @pytest.mark.parametrize("expert_num", EXPERT_NUM) @pytest.mark.parametrize("hidden_size", HIDDEN_DIM) @pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM) @pytest.mark.parametrize("use_bias", USE_BIAS) -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize( + "dtype", + ( + [torch.float32, torch.float16, torch.bfloat16] + if current_platform.get_cpu_architecture() == CpuArchEnum.ARM + else [torch.float32, torch.bfloat16] + ), +) @pytest.mark.parametrize("act", ACT) -@pytest.mark.parametrize("isa", ["neon"]) +@pytest.mark.parametrize( + "isa", + ["neon"] if current_platform.get_cpu_architecture() == CpuArchEnum.ARM else ["vsx"], +) def test_cpu_fused_moe_int8( batch_size: int, expert_num: int, @@ -386,7 +397,8 @@ def test_cpu_fused_moe_int8( isa, ) - torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=2e-2) + atol = rtol = 1e-1 if isa == "vsx" else 2e-2 + torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol) # moe_intermediate_size not a multiple of 32, e.g. what tensor-parallel diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 7b3193fb5f..858b351864 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -950,6 +950,29 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular): # =========================================================================== +def prepare_int8_moe_layer_for_cpu( + w13: torch.Tensor, + w2: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Prepack INT8 MoE weights for the current CPU architecture.""" + # SMMLA packing for AArch64 + if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: + return ( + cpu_prepack_moe_weight_int8(w13, "neon"), + cpu_prepack_moe_weight_int8(w2, "neon"), + ) + # VSX packing for POWER + if current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC: + return ( + cpu_prepack_moe_weight_int8(w13, "vsx"), + cpu_prepack_moe_weight_int8(w2, "vsx"), + ) + # VNNI packing for x86 + packed_w13 = torch.ops._C.convert_weight_packed(w13) + packed_w2 = torch.ops._C.convert_weight_packed(w2) + return packed_w13, packed_w2 + + class CPUExpertsInt8(mk.FusedMoEExpertsModular): """CPU INT8 W8A8 per-channel weight / dynamic per-token activation modular MoE experts.""" @@ -1283,6 +1306,160 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular): ) +class PowerCPUExpertsInt8(mk.FusedMoEExpertsModular): + """POWER VSX INT8 MoE with per-token activation and channelwise weight quant.""" + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def is_supported_config( + cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[bool, str | None]: + supported, reason = mk.FusedMoEExpertsModular.is_supported_config( + cls, + moe_config, + weight_key, + activation_key, + activation_format, + ) + if not supported: + return supported, reason + if moe_config.in_dtype not in (torch.float32, torch.bfloat16): + return False, "kernel requires float32 or bfloat16 activations" + if moe_config.hidden_dim % 16 != 0: + return False, "kernel requires hidden dim divisible by 16" + if moe_config.intermediate_size_per_partition % 16 != 0: + return False, "kernel requires intermediate dim divisible by 16" + return True, None + + @staticmethod + def _supports_current_device() -> bool: + return ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + and hasattr(torch.ops._C, "cpu_fused_moe_int8") + ) + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation in ( + MoEActivation.SILU, + MoEActivation.SWIGLUOAI, + MoEActivation.GELU, + MoEActivation.GELU_TANH, + ) + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return not moe_parallel_config.use_ep + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return (weight_key, activation_key) == ( + kInt8StaticChannelSym, + kInt8DynamicTokenSym, + ) + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + w13 = cpu_prepack_moe_weight_int8(layer.w13_weight, "vsx") + w2 = cpu_prepack_moe_weight_int8(layer.w2_weight, "vsx") + replace_parameter(layer, "w13_weight", w13) + replace_parameter(layer, "w2_weight", w2) + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + # cpu_fused_moe_int8 manages its own scratch space. + return (0,), (0,), (M, K) + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ) -> None: + # The modular prepare step applies router weights to the input when + # apply_router_weight_on_input is enabled. + assert self.w1_scale is not None + assert self.w2_scale is not None + cpu_fused_moe_int8( + output, + hidden_states, + w1, + w2, + self.w1_scale, + self.w2_scale, + self.w1_bias, + self.w2_bias, + topk_weights, + topk_ids, + activation.value, + "vsx", + skip_weighted=apply_router_weight_on_input, + ) + + class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular): """AMD Zen INT8 MoE with per-token activation and channelwise weight quantization, dispatched through zentorch.""" diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index 3b8a3c4f87..1a23b01736 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -71,10 +71,16 @@ def backend_to_kernel_cls( from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( ArmCPUExpertsInt8, CPUExpertsInt8, + PowerCPUExpertsInt8, ZenCPUExpertsInt8, ) - return [ZenCPUExpertsInt8, ArmCPUExpertsInt8, CPUExpertsInt8] + return [ + ZenCPUExpertsInt8, + PowerCPUExpertsInt8, + ArmCPUExpertsInt8, + CPUExpertsInt8, + ] else: raise ValueError(f"Unknown Int8 MoE backend: {backend.value}") From 78fdf4efb86dffde1b0f5733207aed038c3530f8 Mon Sep 17 00:00:00 2001 From: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Date: Thu, 17 Sep 2026 07:00:04 -0700 Subject: [PATCH 031/143] [Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699) Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Signed-off-by: Dakai An Co-authored-by: Dakai An Co-authored-by: OpenAI Codex Co-authored-by: aoshen02 --- tests/kernels/moe/test_flashinfer.py | 258 ++++++++++++++++++ .../moe/test_moe_weight_loading_padded.py | 126 +++++++++ tests/kernels/moe/test_trtllm_bf16_moe.py | 8 +- .../model_loader/test_reload.py | 105 +++++++ .../layers/fused_moe/routed_experts.py | 18 ++ .../fused_moe/unquantized_fused_moe_method.py | 122 +++++++-- .../quantization/utils/flashinfer_utils.py | 86 +++--- .../model_loader/reload/meta.py | 4 +- .../model_loader/reload/utils.py | 8 +- 9 files changed, 667 insertions(+), 68 deletions(-) diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index 2e022ede36..53d760c339 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import multiprocessing from dataclasses import dataclass import pytest @@ -433,6 +434,263 @@ def test_convert_moe_weights_to_flashinfer_trtllm_block_layout( assert w13_converted.shape[0] == num_experts assert w2_converted.shape[0] == num_experts + assert w13_converted.data_ptr() == w13.data_ptr() + assert w2_converted.data_ptr() == w2.data_ptr() + + +@pytest.mark.parametrize("is_gated_act_gemm", [True, False]) +def test_convert_moe_weights_to_flashinfer_trtllm_block_layout_values( + is_gated_act_gemm, +): + from flashinfer.fused_moe.core import ( + _maybe_get_cached_w3_w1_permute_indices, + get_w2_permute_indices_with_cache, + ) + + from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + convert_moe_weights_to_flashinfer_trtllm_block_layout, + ) + + num_experts, intermediate, hidden = 2, 256, 256 + w13_multiplier = 2 if is_gated_act_gemm else 1 + w13 = torch.randn( + (num_experts, w13_multiplier * intermediate, hidden), + dtype=torch.bfloat16, + device="cuda", + ) + w2 = torch.randn( + (num_experts, hidden, intermediate), + dtype=torch.bfloat16, + device="cuda", + ) + + def _reference_block_layout( + weight: torch.Tensor, + is_w13: bool, + cache: dict[torch.Size, torch.Tensor], + ) -> torch.Tensor: + outputs = [] + for expert in weight: + expert_uint8 = expert.view(torch.uint8) + if is_w13: + indices = _maybe_get_cached_w3_w1_permute_indices( + cache, + expert_uint8, + 128, + is_gated_act_gemm=is_gated_act_gemm, + ) + if is_gated_act_gemm: + indices = (indices + expert_uint8.shape[0] // 2) % ( + expert_uint8.shape[0] + ) + else: + indices = get_w2_permute_indices_with_cache( + cache, + expert_uint8, + 128, + ) + rows, cols = expert_uint8.shape + blocks = expert_uint8.view(rows, cols // 128, 128).permute(1, 0, 2) + outputs.append(torch.index_select(blocks, 1, indices.to(weight.device))) + return torch.stack(outputs).view(torch.bfloat16) + + reference_cache: dict[torch.Size, torch.Tensor] = {} + expected_w13 = _reference_block_layout(w13, is_w13=True, cache=reference_cache) + expected_w2 = _reference_block_layout(w2, is_w13=False, cache=reference_cache) + w13_ptr = w13.data_ptr() + w2_ptr = w2.data_ptr() + + actual_w13, actual_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout( + {}, + w13, + w2, + is_gated_act_gemm=is_gated_act_gemm, + ) + + assert actual_w13.data_ptr() == w13_ptr + assert actual_w2.data_ptr() == w2_ptr + assert torch.equal(actual_w13, expected_w13) + assert torch.equal(actual_w2, expected_w2) + + +def _make_unquantized_flashinfer_test_layer( + monkeypatch, intermediate, is_gated, *, device="cuda" +): + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, + ) + from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( + UnquantizedFusedMoEMethod, + ) + + moe_config = make_dummy_moe_config( + num_experts=2, + hidden_dim=256, + intermediate_size=intermediate, + activation=MoEActivation.SILU if is_gated else MoEActivation.RELU2_NO_MUL, + ) + moe_config.intermediate_size_per_partition = (intermediate + 127) // 128 * 128 + method = object.__new__(UnquantizedFusedMoEMethod) + method.moe = moe_config + method.unquantized_backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM + method.moe_kernel = None + monkeypatch.setattr( + method, + "_init_moe_kernel", + lambda _: setattr(method, "moe_kernel", object()), + ) + + layer = torch.nn.Module() + layer.moe_config = moe_config + with torch.device(device): + method.create_weights( + layer, + num_experts=2, + hidden_size=256, + intermediate_size_per_partition=moe_config.intermediate_size_per_partition, + params_dtype=torch.bfloat16, + ) + return method, layer + + +@pytest.mark.parametrize("intermediate", [192, 256]) +@pytest.mark.parametrize("is_gated", [True, False]) +def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed( + monkeypatch, intermediate, is_gated +): + """Each raw reload clears padding left dirty by prior in-place conversion.""" + from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + convert_moe_weights_to_flashinfer_trtllm_block_layout, + ) + + method, layer = _make_unquantized_flashinfer_test_layer( + monkeypatch, intermediate, is_gated + ) + w13_shape, w2_shape = layer.w13_weight.shape, layer.w2_weight.shape + padded = w2_shape[-1] + w13_ptr = layer.w13_weight.data_ptr() + w2_ptr = layer.w2_weight.data_ptr() + + for _ in range(2): + reloaded_w13 = torch.randn_like(layer.w13_weight) + reloaded_w2 = torch.randn_like(layer.w2_weight) + reloaded_w13[:, intermediate:padded].zero_() + if is_gated: + reloaded_w13[:, padded + intermediate :].zero_() + reloaded_w2[:, :, intermediate:].zero_() + expected_w13, expected_w2 = ( + convert_moe_weights_to_flashinfer_trtllm_block_layout( + {}, + reloaded_w13.clone(), + reloaded_w2.clone(), + is_gated_act_gemm=is_gated, + ) + ) + + # The loader writes only logical checkpoint slices, not padding. + layer.w13_weight.fill_(float("nan")) + layer.w2_weight.fill_(float("nan")) + layer.w13_weight[:, :intermediate].copy_(reloaded_w13[:, :intermediate]) + if is_gated: + layer.w13_weight[:, padded : padded + intermediate].copy_( + reloaded_w13[:, padded : padded + intermediate] + ) + layer.w2_weight[:, :, :intermediate].copy_(reloaded_w2[:, :, :intermediate]) + method.process_weights_after_loading(layer) + + assert layer.w13_weight.shape == w13_shape + assert layer.w2_weight.shape == w2_shape + assert layer.w13_weight.data_ptr() == w13_ptr + assert layer.w2_weight.data_ptr() == w2_ptr + kernel_w13, kernel_w2 = method._kernel_weights(layer) + assert torch.equal(kernel_w13, expected_w13) + assert torch.equal(kernel_w2, expected_w2) + + +def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_index): + from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader + from vllm.model_executor.utils import weights_already_processed + + torch.accelerator.set_device_index(device_index) + with pytest.MonkeyPatch.context() as monkeypatch: + method, layer = _make_unquantized_flashinfer_test_layer( + monkeypatch, 192, is_gated, device="meta" + ) + loader = object.__new__(IpcModelLoader) + loader.mode = mode + loader._apply_entries(layer, entries, {}, device_index) + pointers = (layer.w13_weight.data_ptr(), layer.w2_weight.data_ptr()) + + # Like ipc_cache: a fresh method, only tensor metadata, no _setup_kernel. + with weights_already_processed(): + method.process_weights_after_loading(layer) + assert method.moe_kernel is not None + actual = method._kernel_weights(layer) + for weight, reference, pointer in zip(actual, expected, pointers): + assert weight.data_ptr() == pointer + assert torch.equal(weight.cpu(), reference) + # Cross-process addresses need not match; writes prove sharing/isolation. + with torch.no_grad(): + for weight in actual: + weight.fill_(1.0) + torch.accelerator.synchronize() + + +@pytest.mark.parametrize("is_gated", [True, False]) +@pytest.mark.parametrize("cache_ndim", [3, 4]) +@pytest.mark.parametrize("mode", ["copy", "zero_copy"]) +def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state( + monkeypatch, is_gated, cache_ndim, mode +): + """A spawned IPC consumer restores views without transient method state.""" + from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + convert_moe_weights_to_flashinfer_trtllm_block_layout, + ) + from vllm.model_executor.model_loader.weight_cache.protocol import TensorEntry + + method, layer = _make_unquantized_flashinfer_test_layer( + monkeypatch, 192, is_gated, device="meta" + ) + packed = convert_moe_weights_to_flashinfer_trtllm_block_layout( + {}, + torch.randn_like(layer.w13_weight, device="cuda"), + torch.randn_like(layer.w2_weight, device="cuda"), + is_gated_act_gemm=is_gated, + ) + expected = [weight.cpu() for weight in packed] + entries = {} + for name, weight in zip(("w13_weight", "w2_weight"), packed): + cached = weight.view_as(getattr(layer, name)) if cache_ndim == 3 else weight + entries[name] = TensorEntry.from_tensor(cached, kind="param") + consumer = multiprocessing.get_context("spawn").Process( + target=_check_flashinfer_ipc_weights, + args=( + entries, + expected, + is_gated, + mode, + torch.accelerator.current_device_index(), + ), + ) + consumer.start() + try: + # Keep producer allocations alive until the consumer releases its views. + consumer.join(timeout=180) + assert not consumer.is_alive(), "IPC consumer timed out" + assert consumer.exitcode == 0, "IPC consumer failed; see child traceback" + for source, reference in zip(packed, expected): + expected_source = ( + torch.ones_like(reference) if mode == "zero_copy" else reference + ) + assert torch.equal(source.cpu(), expected_source) + finally: + if consumer.is_alive(): + consumer.terminate() + consumer.join(timeout=10) + if consumer.is_alive(): + consumer.kill() + consumer.join(timeout=10) + consumer.close() @pytest.mark.parametrize( diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py index d3716a196b..423f181132 100644 --- a/tests/kernels/moe/test_moe_weight_loading_padded.py +++ b/tests/kernels/moe/test_moe_weight_loading_padded.py @@ -12,7 +12,14 @@ correctly handles this mismatch. import pytest import torch +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, +) from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts +from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( + UnquantizedFusedMoEMethod, +) from .utils import make_dummy_moe_config @@ -249,6 +256,57 @@ class TestNarrowExpertDataForPadding: class TestWeightLoadingWithPaddedHiddenSize: """Integration-style tests that simulate padded weight loading.""" + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + @pytest.mark.parametrize("tp_rank", [0, 1]) + def test_load_w2_chunks_noncontiguous_cpu_source_to_padded_cuda( + self, dtype, tp_rank + ): + hidden = 4096 + intermediate = 1024 + loaded_weight = ( + torch.arange(hidden * intermediate * 2) + .remainder(251) + .to(dtype) + .reshape(hidden, intermediate * 2) + ) + tp_source = loaded_weight[ + :, intermediate * tp_rank : intermediate * (tp_rank + 1) + ] + expert_data_full = torch.zeros( + hidden + 8, intermediate + 8, device="cuda", dtype=dtype + ) + destination = expert_data_full[:hidden, :intermediate] + + assert not tp_source.is_contiguous() + assert tp_source.nbytes > 1 << 20 + assert not destination.is_contiguous() + + experts = object.__new__(RoutedExperts) + torch.nn.Module.__init__(experts) + experts.moe_config = make_dummy_moe_config() + experts.moe_config.moe_parallel_config.tp_size = 2 + + torch.accelerator.synchronize() + allocated_before = torch.accelerator.memory_allocated() + torch.accelerator.reset_peak_memory_stats() + experts._load_w2( + expert_data=expert_data_full, + shard_dim=1, + loaded_weight=loaded_weight, + tp_rank=tp_rank, + ) + torch.accelerator.synchronize() + peak_extra = torch.accelerator.max_memory_allocated() - allocated_before + + # A single copy into the strided CUDA view allocates a full-shard + # temporary. Correct values alone would not catch that regression. + assert peak_extra < tp_source.nbytes + + torch.testing.assert_close(destination.cpu(), tp_source) + assert torch.count_nonzero(expert_data_full[hidden:, :]) == 0 + assert torch.count_nonzero(expert_data_full[:hidden, intermediate:]) == 0 + def test_load_w2_with_padding(self): """Simulate loading w2 weights when hidden_size is padded.""" padded_hidden = 3072 @@ -364,6 +422,74 @@ class TestWeightLoadingWithPaddedHiddenSize: ) +class TestUnquantizedTrtLlmPrePadding: + @staticmethod + def _make_method( + intermediate: int, + backend: UnquantizedMoeBackend = UnquantizedMoeBackend.FLASHINFER_TRTLLM, + ) -> UnquantizedFusedMoEMethod: + moe_config = make_dummy_moe_config( + num_experts=2, + hidden_dim=64, + intermediate_size=intermediate, + ) + method = object.__new__(UnquantizedFusedMoEMethod) + method.moe = moe_config + method.unquantized_backend = backend + method.moe_kernel = None + return method + + @pytest.mark.parametrize( + "backend,original,expected", + [ + (UnquantizedMoeBackend.FLASHINFER_TRTLLM, 1344, 1408), + (UnquantizedMoeBackend.FLASHINFER_TRTLLM, 1408, 1408), + (UnquantizedMoeBackend.TRITON, 1344, 1344), + ], + ) + def test_rounds_intermediate_before_weight_allocation( + self, + backend: UnquantizedMoeBackend, + original: int, + expected: int, + ): + method = self._make_method(original, backend) + + hidden, intermediate = method.maybe_roundup_sizes( + hidden_size=64, + intermediate_size_per_partition=original, + act_dtype=torch.bfloat16, + moe_parallel_config=method.moe.moe_parallel_config, + ) + + assert hidden == 64 + assert intermediate == expected + + @pytest.mark.parametrize("is_act_and_mul", [True, False]) + def test_create_weights_zeroes_preallocated_padding(self, is_act_and_mul: bool): + method = self._make_method(1344) + if not is_act_and_mul: + method.moe.activation = MoEActivation.RELU2_NO_MUL + method.moe.intermediate_size_per_partition = 1408 + layer = torch.nn.Module() + + method.create_weights( + layer=layer, + num_experts=2, + hidden_size=64, + intermediate_size_per_partition=1408, + params_dtype=torch.bfloat16, + ) + + up_mult = 2 if is_act_and_mul else 1 + assert layer.w13_weight.shape == (2, up_mult * 1408, 64) + assert layer.w2_weight.shape == (2, 64, 1408) + assert torch.count_nonzero(layer.w13_weight[:, 1344:1408]) == 0 + if is_act_and_mul: + assert torch.count_nonzero(layer.w13_weight[:, 2752:]) == 0 + assert torch.count_nonzero(layer.w2_weight[:, :, 1344:]) == 0 + + class TestLoadWeightsExpertBias: """Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize all-zero per-expert `.bias` tensors for models whose experts have no bias diff --git a/tests/kernels/moe/test_trtllm_bf16_moe.py b/tests/kernels/moe/test_trtllm_bf16_moe.py index 9e50b4fc3b..f7466c63c8 100644 --- a/tests/kernels/moe/test_trtllm_bf16_moe.py +++ b/tests/kernels/moe/test_trtllm_bf16_moe.py @@ -75,6 +75,10 @@ def test_trtllm_bf16_moe_modular_no_graph( a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + # The FlashInfer conversion may rewrite unpadded input storage in place. + # Preserve the original layout for the independent torch reference. + reference_w1 = w1.clone() + reference_w2 = w2.clone() score = torch.randn((m, e), device="cuda", dtype=dtype) scores = torch.softmax(score, dim=-1, dtype=torch.float32) topk_weights, topk_ids = torch.topk(scores, topk) @@ -133,8 +137,8 @@ def test_trtllm_bf16_moe_modular_no_graph( torch_output = torch_moe( a, - w1, - w2, + reference_w1, + reference_w2, score, topk, activation=MoEActivation.SILU, diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index b4a544d2fc..844149ce1b 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -5,6 +5,7 @@ import importlib.machinery import inspect import sys import types +from types import SimpleNamespace from unittest.mock import Mock from weakref import WeakKeyDictionary, ref @@ -467,6 +468,110 @@ def test_model_cleanup(dist_init, default_vllm_config): assert len(mock_info_dict) == 0 +@pytest.mark.parametrize("is_gated", [False, True]) +@pytest.mark.parametrize("has_bias", [False, True]) +@pytest.mark.parametrize("tp_rank", [0, 1]) +@pytest.mark.parametrize("padded", [False, True]) +def test_padded_moe_reload_releases_each_layer( + monkeypatch, is_gated, has_bias, tp_rank, padded +): + """Checkpoint-sized copies finish each layer without global finalization.""" + from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts + from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( + UnquantizedFusedMoEMethod, + ) + + hidden, intermediate, experts = 4, 3, 2 + stored_hidden, stored_intermediate = (8, 8) if padded else (hidden, intermediate) + config = SimpleNamespace( + hidden_dim_unpadded=hidden, + intermediate_size_per_partition_unpadded=intermediate, + is_act_and_mul=is_gated, + has_bias=has_bias, + tp_rank=tp_rank, + moe_parallel_config=SimpleNamespace(tp_size=2), + ) + model = torch.nn.ModuleList() + processed: list[torch.nn.Module] = [] + for _ in range(2): + method = object.__new__(UnquantizedFusedMoEMethod) + torch.nn.Module.__init__(method) + method.moe = config + # The regression concerns streaming reload, not kernel conversion. + monkeypatch.setattr(method, "process_weights_after_loading", processed.append) + layer = object.__new__(RoutedExperts) + torch.nn.Module.__init__(layer) + layer.moe_config = config + layer.quant_config = None + layer.quant_method = method + layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i) + layer._loaded_expert_biases = set() + method.create_weights( + layer, + experts, + stored_hidden, + stored_intermediate, + torch.float32, + weight_loader=layer.weight_loader, + ) + model.append(layer) + + record_metadata_for_reloading(model) + original_params = [dict(layer.named_parameters()) for layer in model] + shards = ["w1", "w3", "w2"] if is_gated else ["w1", "w2"] + for cycle in range(2): + initialize_layerwise_reload(model) + for layer_index, layer in enumerate(model): + info = reload_layerwise.get_layerwise_info(layer) + inputs = [] + expected = { + name: torch.full_like(p, float("nan")) + for name, p in original_params[layer_index].items() + } + params = dict(layer.named_parameters()) + calls = [ + (e, s, b) + for e in range(experts) + for s in shards + for b in ([False, True] if has_bias else [False]) + ] + for call_index, (expert, shard, bias) in enumerate(calls): + name = ("w2" if shard == "w2" else "w13") + ( + "_bias" if bias else "_weight" + ) + shape = ( + ((hidden,) if bias else (hidden, 2 * intermediate)) + if shard == "w2" + else ((2 * intermediate,) if bias else (2 * intermediate, hidden)) + ) + weight = torch.arange( + torch.Size(shape).numel(), dtype=torch.float32 + ).reshape(shape) + weight = weight + 100 * (1 + call_index + cycle) + inputs.append(ref(weight)) + # Direct checkpoint loading is the reference for deferred reload. + layer.weight_loader(expected[name], weight, name, shard, expert) + param = params[name] + param.weight_loader(param, weight, name, shard, expert) + del weight + if call_index != len(calls) - 1: + assert info.can_load(), ( + "Layer processed before its final checkpoint shard" + ) + + assert not info.can_load(), ( + "Padding must not defer the layer until finalization" + ) + assert not info.loaded_weights + assert len(processed) == cycle * len(model) + layer_index + 1 + assert all(source() is None for source in inputs) + for name, original in original_params[layer_index].items(): + assert getattr(layer, name) is original + # Kernel-specific tests cover padding, which is not checkpoint data. + mask = torch.isfinite(expected[name]) + assert torch.equal(original[mask], expected[name][mask]) + + def test_get_numel_loaded(): param = torch.empty(10, device="meta") loaded_weight = torch.empty(10) diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 42ea15dcda..e3a287ddc2 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, resolve_quant_method, ) +from vllm.utils.math_utils import cdiv if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts @@ -566,6 +567,23 @@ class RoutedExperts(PluggableLayer): hidden_dim=hidden_dim, shard_dim=shard_dim, ) + if ( + loaded_weight.device.type == "cpu" + and expert_data.device.type == "cuda" + and not expert_data.is_contiguous() + and expert_data.ndim == 2 + ): + # A strided CPU-to-CUDA copy can allocate a full-shard CUDA + # temporary. Make the TP slice contiguous on CPU and copy rows + # in chunks to limit the temporary used for the padded view. + loaded_weight = loaded_weight.contiguous() + num_chunks = max(cdiv(loaded_weight.nbytes, 1 << 20), 1) + for dst, src in zip( + expert_data.chunk(num_chunks, dim=0), + loaded_weight.chunk(num_chunks, dim=0), + ): + dst.copy_(src) + return expert_data.copy_(loaded_weight) def _load_single_value( diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index 523d1df0d8..a1b2471f5d 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -11,6 +11,7 @@ from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, + FusedMoEParallelConfig, FusedMoEQuantConfig, biased_moe_quant_config, ) @@ -33,6 +34,7 @@ from vllm.model_executor.utils import ( set_weight_attrs, ) from vllm.platforms import current_platform +from vllm.utils.math_utils import round_up if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts @@ -68,22 +70,38 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): params_dtype: torch.dtype, **extra_weight_attrs, ): + unpadded_intermediate = self.moe.intermediate_size_per_partition_unpadded + assert unpadded_intermediate is not None + has_intermediate_padding = ( + intermediate_size_per_partition > unpadded_intermediate + ) if self.moe.is_act_and_mul: w13_up_dim = 2 * intermediate_size_per_partition else: w13_up_dim = intermediate_size_per_partition + unpadded_hidden = self.moe.hidden_dim_unpadded + assert unpadded_hidden is not None + unpadded_up = unpadded_intermediate * (2 if self.moe.is_act_and_mul else 1) # Fused gate_up_proj (column parallel) - w13_weight = torch.nn.Parameter( - torch.empty( - num_experts, - w13_up_dim, - hidden_size, - dtype=params_dtype, - ), - requires_grad=False, + w13_data = torch.empty( + num_experts, + w13_up_dim, + hidden_size, + dtype=params_dtype, ) + if has_intermediate_padding: + # The checkpoint loader only writes the unpadded slices. Zero the + # tails now so no full-size padding copy is needed after loading. + w13_data[:, unpadded_intermediate:intermediate_size_per_partition].zero_() + if self.moe.is_act_and_mul: + w13_data[ + :, + intermediate_size_per_partition + unpadded_intermediate :, + ].zero_() + w13_weight = torch.nn.Parameter(w13_data, requires_grad=False) layer.register_parameter("w13_weight", w13_weight) set_weight_attrs(w13_weight, extra_weight_attrs) + w13_weight.weight_loader_numel = num_experts * unpadded_up * unpadded_hidden if self.moe.has_bias: w13_bias = torch.nn.Parameter( torch.zeros(num_experts, w13_up_dim, dtype=params_dtype), @@ -91,18 +109,22 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ) layer.register_parameter("w13_bias", w13_bias) set_weight_attrs(w13_bias, extra_weight_attrs) + w13_bias.weight_loader_numel = num_experts * unpadded_up # down_proj (row parallel) - w2_weight = torch.nn.Parameter( - torch.empty( - num_experts, - hidden_size, - intermediate_size_per_partition, - dtype=params_dtype, - ), - requires_grad=False, + w2_data = torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, ) + if has_intermediate_padding: + w2_data[:, :, unpadded_intermediate:].zero_() + w2_weight = torch.nn.Parameter(w2_data, requires_grad=False) layer.register_parameter("w2_weight", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) + w2_weight.weight_loader_numel = ( + num_experts * unpadded_hidden * unpadded_intermediate + ) if self.moe.has_bias: w2_bias = torch.nn.Parameter( torch.zeros(num_experts, hidden_size, dtype=params_dtype), @@ -110,6 +132,26 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ) layer.register_parameter("w2_bias", w2_bias) set_weight_attrs(w2_bias, extra_weight_attrs) + w2_bias.weight_loader_numel = num_experts * unpadded_hidden + + def maybe_roundup_sizes( + self, + hidden_size: int, + intermediate_size_per_partition: int, + act_dtype: torch.dtype, + moe_parallel_config: FusedMoEParallelConfig, + ) -> tuple[int, int]: + hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes( + hidden_size, + intermediate_size_per_partition, + act_dtype, + moe_parallel_config, + ) + if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: + intermediate_size_per_partition = round_up( + intermediate_size_per_partition, 128 + ) + return hidden_size, intermediate_size_per_partition def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor: # Pad the weight tensor. This is an optimization on ROCm platform, which @@ -135,6 +177,21 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): w13: torch.Tensor, w2: torch.Tensor, ) -> None: + w13_parameter_shape = w13.shape + w2_parameter_shape = w2.shape + + if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: + unpadded = self.moe.intermediate_size_per_partition_unpadded + assert unpadded is not None + intermediate = w2.shape[-1] + # Reloads overwrite only checkpoint slices; the previous in-place + # permutation can leave nonzero values in the raw padding slots. + if intermediate > unpadded: + w13[:, unpadded:intermediate].zero_() + if self.moe.is_act_and_mul: + w13[:, intermediate + unpadded :].zero_() + w2[:, :, unpadded:].zero_() + # Shuffle weights to runtime format. w13_new, w2_new = convert_to_unquantized_kernel_format( self.unquantized_backend, @@ -142,6 +199,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): w13_weight=w13, w2_weight=w2, ) + if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: + w13_new = w13_new.view(w13_parameter_shape) + w2_new = w2_new.view(w2_parameter_shape) + # `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__; # On the first call we replace the parameter normally. On subsequent # calls (e.g. RL weight updates that re-trigger @@ -170,6 +231,25 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): assert self.moe_kernel is not None self.moe_kernel.fused_experts.process_weights_after_loading(layer) + def _kernel_weights( + self, layer: "RoutedExperts" + ) -> tuple[torch.Tensor, torch.Tensor]: + if self.unquantized_backend != UnquantizedMoeBackend.FLASHINFER_TRTLLM: + return layer.w13_weight, layer.w2_weight + + # IPC caches restore tensor shapes/storage, not method attributes. + # BlockMajorK uses 128-byte blocks, i.e. 64 BF16 elements. + def block_view(weight: torch.Tensor) -> torch.Tensor: + if weight.ndim == 4: + return weight + experts, rows, cols = weight.shape + return weight.view(experts, cols // 64, rows, 64) + + return ( + block_view(layer.w13_weight), + block_view(layer.w2_weight), + ) + def _init_moe_kernel(self, layer: "RoutedExperts") -> None: """Build the MoE kernel from the layer's current (shuffled) weights.""" self.moe_quant_config = self.get_fused_moe_quant_config(layer) @@ -274,10 +354,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): shared_experts_input: torch.Tensor | None, ) -> torch.Tensor: assert self.moe_kernel is not None + w13, w2 = self._kernel_weights(layer) return self.moe_kernel.apply( hidden_states=x, - w1=layer.w13_weight, - w2=layer.w2_weight, + w1=w13, + w2=w2, topk_weights=topk_weights, topk_ids=topk_ids, activation=layer.activation, @@ -315,10 +396,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.is_monolithic assert self.moe_kernel is not None + w13, w2 = self._kernel_weights(layer) return self.moe_kernel.apply_monolithic( x, - layer.w13_weight, - layer.w2_weight, + w13, + w2, router_logits, activation=layer.activation, global_num_experts=layer.global_num_experts, diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py index 044285ac85..1bfd7e61f9 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py @@ -112,8 +112,9 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout( ) -> tuple[torch.Tensor, torch.Tensor]: """Convert expert weights to FlashInfer's block layout. - This reorders W13 and W2 into the expected epilogue-tiled block layout and - returns the shuffled weight tensors. + This reorders W13 and W2 in place into the expected epilogue-tiled block + layout and returns views of the shuffled weight tensors. Using one expert + as scratch space avoids allocating another full copy of both weights. """ if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16: raise ValueError( @@ -147,51 +148,50 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout( out=out, ) - w13_rows, w13_cols = w13_weight[0].view(torch.uint8).shape - w2_rows, w2_cols = w2_weight[0].view(torch.uint8).shape - w13_weights_shuffled_tensor = torch.empty( - (num_experts, w13_cols // block_k, w13_rows, block_k), - dtype=torch.uint8, - device=w13_weight.device, - ) - w2_weights_shuffled_tensor = torch.empty( - (num_experts, w2_cols // block_k, w2_rows, block_k), - dtype=torch.uint8, - device=w2_weight.device, - ) - - for i in range(num_experts): - w13_expert_uint8 = w13_weight[i].view(torch.uint8) - - permute_indices = _maybe_get_cached_w3_w1_permute_indices( - cache_permute_indices, - w13_expert_uint8, - epilogue_tile_m, - is_gated_act_gemm=is_gated_act_gemm, - ) - if is_gated_act_gemm: - rows = w13_expert_uint8.shape[0] - permute_indices = (permute_indices + rows // 2) % rows - _copy_permuted_expert_to_block_layout( - w13_weights_shuffled_tensor[i], - w13_expert_uint8, - permute_indices, + def _convert_weight_in_place( + weight: torch.Tensor, + is_w13: bool, + ) -> torch.Tensor: + rows, cols = weight[0].view(torch.uint8).shape + block_layout_shape = (num_experts, cols // block_k, rows, block_k) + expert_scratch = torch.empty( + block_layout_shape[1:], + dtype=torch.uint8, + device=weight.device, ) - permute_indices = get_w2_permute_indices_with_cache( - cache_permute_indices, - w2_weight[i].view(torch.uint8), - epilogue_tile_m, - ) - _copy_permuted_expert_to_block_layout( - w2_weights_shuffled_tensor[i], - w2_weight[i].view(torch.uint8), - permute_indices, - ) + for i in range(num_experts): + expert_uint8 = weight[i].view(torch.uint8) + if is_w13: + permute_indices = _maybe_get_cached_w3_w1_permute_indices( + cache_permute_indices, + expert_uint8, + epilogue_tile_m, + is_gated_act_gemm=is_gated_act_gemm, + ) + if is_gated_act_gemm: + permute_indices = ( + permute_indices + expert_uint8.shape[0] // 2 + ) % expert_uint8.shape[0] + else: + permute_indices = get_w2_permute_indices_with_cache( + cache_permute_indices, + expert_uint8, + epilogue_tile_m, + ) + + _copy_permuted_expert_to_block_layout( + expert_scratch, + expert_uint8, + permute_indices, + ) + expert_uint8.view(-1).copy_(expert_scratch.view(-1)) + + return weight.view(torch.uint8).view(block_layout_shape).view(torch.bfloat16) return ( - w13_weights_shuffled_tensor.view(torch.bfloat16), - w2_weights_shuffled_tensor.view(torch.bfloat16), + _convert_weight_in_place(w13_weight, is_w13=True), + _convert_weight_in_place(w2_weight, is_w13=False), ) diff --git a/vllm/model_executor/model_loader/reload/meta.py b/vllm/model_executor/model_loader/reload/meta.py index 4d08d49165..9874ebd67b 100644 --- a/vllm/model_executor/model_loader/reload/meta.py +++ b/vllm/model_executor/model_loader/reload/meta.py @@ -9,7 +9,7 @@ from torch.utils._python_dispatch import TorchDispatchMode from .sanitize import restore_layer_refs, sanitize_layer_refs from .types import LayerReloadingInfo, LayerTensors -from .utils import get_layer_params_buffers, get_layer_tensors +from .utils import get_layer_params_buffers, get_layer_tensors, get_tensor_load_numel __all__ = [ "to_meta_tensor", @@ -206,5 +206,5 @@ def get_numel_loaded( numel = counter.copied_numel param = args.arguments.get("param", None) if isinstance(param, torch.Tensor): - numel = min(numel, param.numel()) + numel = min(numel, get_tensor_load_numel(param)) return numel, return_value diff --git a/vllm/model_executor/model_loader/reload/utils.py b/vllm/model_executor/model_loader/reload/utils.py index b7dc361503..203aa82af3 100644 --- a/vllm/model_executor/model_loader/reload/utils.py +++ b/vllm/model_executor/model_loader/reload/utils.py @@ -11,6 +11,7 @@ __all__ = [ "get_layer_tensors", "get_layer_params_buffers", "get_layer_size", + "get_tensor_load_numel", "has_device_tensors", "get_info_size", ] @@ -30,6 +31,11 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors: ) +def get_tensor_load_numel(tensor: torch.Tensor) -> int: + """Count checkpoint elements, excluding padding declared by the weight creator.""" + return getattr(tensor, "weight_loader_numel", tensor.numel()) + + def get_layer_size(layer: torch.nn.Module) -> int: """Calculate total number of elements across loadable tensors in a layer. @@ -39,7 +45,7 @@ def get_layer_size(layer: torch.nn.Module) -> int: from .meta import SKIP_LOAD_TENSORS return sum( - tensor.numel() + get_tensor_load_numel(tensor) for name, tensor in get_layer_tensors(layer).items() if name not in SKIP_LOAD_TENSORS ) From 861a299adc72bbbb1b60853b6173075465a6b425 Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:11:14 -0400 Subject: [PATCH 032/143] [Bugfix][Model Runner V2] Route dummy tokens to MoE experts during profiling (#57270) Signed-off-by: Robert Shaw Co-authored-by: Claude Opus 5 (1M context) --- tests/v1/worker/test_gpu_input_batch_v2.py | 25 ++++++++++++++++++++++ vllm/v1/worker/gpu/input_batch.py | 3 ++- vllm/v1/worker/gpu/model_runner.py | 3 +++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/v1/worker/test_gpu_input_batch_v2.py b/tests/v1/worker/test_gpu_input_batch_v2.py index 8937f2dc18..13a92d2c0f 100644 --- a/tests/v1/worker/test_gpu_input_batch_v2.py +++ b/tests/v1/worker/test_gpu_input_batch_v2.py @@ -6,6 +6,9 @@ import numpy as np import pytest import torch +from vllm.config import VllmConfig +from vllm.forward_context import set_forward_context +from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk from vllm.platforms import current_platform from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens from vllm.v1.worker.gpu import cp_utils @@ -56,6 +59,28 @@ def test_make_dummy_distributes_remainder(num_reqs: int, num_tokens: int): ) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA top-k.") +@pytest.mark.parametrize("is_padding", [True, False]) +def test_make_dummy_padding_controls_moe_routing(monkeypatch, is_padding: bool): + """Dummy tokens marked as padding are dropped by the MoE router (top-k id + -1), which is wanted for idle DP ranks. Profile runs must not mark them, or + no token reaches the experts and MoE memory is never profiled.""" + monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1") + num_tokens = 16 + buffers = InputBuffers( + max_num_reqs=4, max_num_tokens=num_tokens, device=torch.device(DEVICE) + ) + batch = InputBatch.make_dummy(4, num_tokens, buffers, is_padding=is_padding) + hidden_states = torch.randn(num_tokens, 4, device=DEVICE) + router_logits = torch.randn(num_tokens, 8, device=DEVICE) + + with set_forward_context(None, VllmConfig(), is_padding=batch.is_padding): + _, topk_ids, _ = fused_topk(hidden_states, router_logits, 2, False) + + assert bool((topk_ids == -1).all()) is is_padding + assert bool((topk_ids >= 0).all()) is not is_padding + + def test_maybe_prepare_dcp_local_seq_lens_uses_shared_buffer(monkeypatch): """The batch must view the caller-owned buffer, sliced to padded length. diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index d3c65bd06f..32c083e459 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -125,6 +125,7 @@ class InputBatch: num_tokens: int, input_buffers: InputBuffers, max_query_len: int | None = None, + is_padding: bool = True, ) -> "InputBatch": assert 0 < num_reqs <= num_tokens device = input_buffers.device @@ -168,7 +169,7 @@ class InputBatch: input_ids = input_buffers.input_ids[:num_tokens].zero_() positions = input_buffers.positions[:num_tokens].zero_() - input_buffers.is_padding[:num_tokens].fill_(True) + input_buffers.is_padding[:num_tokens].fill_(is_padding) is_padding = input_buffers.is_padding[:num_tokens] logits_indices = query_start_loc[1:] - 1 diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 97571df8a9..631e6b04d4 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1733,6 +1733,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): batch_desc.num_tokens, self.input_buffers, max_query_len=batch_desc.max_query_len, + # Profiling and warmup must route the dummy tokens to experts + # so MoE memory is measured and MoE kernels are exercised. + is_padding=not is_profile, ) if self.pcp_manager is not None: input_batch = self.pcp_manager.prepare_inputs_to_capture(input_batch) From 9e1cd262cbf43b3f3839a28cfa151e2e4b27e2d8 Mon Sep 17 00:00:00 2001 From: Alexander Lee Date: Thu, 17 Sep 2026 10:44:36 -0400 Subject: [PATCH 033/143] [Bugfix] Use DP index for dense DP weight updates and EC CPU region (#56950) Signed-off-by: lxy-alexander Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5 Co-authored-by: aoshen02 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../unit/cpu/test_ec_shared_region.py | 28 +++++++++++++++++++ .../worker/test_gpu_worker_weight_transfer.py | 18 +++++++++--- .../ec_transfer/ec_connector/cpu/common.py | 4 +-- vllm/v1/worker/gpu_worker.py | 2 +- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py b/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py index 722c0d89fa..c168970147 100644 --- a/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py +++ b/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py @@ -13,6 +13,8 @@ from unittest.mock import MagicMock, patch import pytest import torch +from vllm.config import ParallelConfig +from vllm.distributed.ec_transfer.ec_connector.cpu import common from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( ECSharedRegion, _wait_for_file_size, @@ -95,6 +97,32 @@ def test_second_instance_opens_existing_file_and_shares_memory(): r1.cleanup() +def test_dense_dp_engines_get_distinct_regions(monkeypatch): + """Dense DP engines reset data_parallel_rank to 0 but share instance_id, + so the region must be keyed by data_parallel_index.""" + monkeypatch.setattr(common, "_get_encoder_cache_hidden_dim", lambda cfg: 8) + instance_id = str(uuid.uuid4()) + regions = [] + try: + for dp_index in range(2): + parallel_config = ParallelConfig( + data_parallel_size=2, data_parallel_rank=dp_index + ) + parallel_config.reconfigure_for_independent_dp_rank() + cfg = MagicMock() + cfg.instance_id = instance_id + cfg.parallel_config = parallel_config + cfg.model_config.dtype = torch.float16 + cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64} + regions.append(common.create_ec_shared_region(cfg)) + + assert all(r._is_creator for r in regions) + assert regions[0]._mmap_path != regions[1]._mmap_path + finally: + for r in regions: + r.cleanup() + + def test_only_creator_unlinks_file_on_cleanup(): """Critical contract: if the non-creator unlinks, the creator's mmap path becomes a dangling backing file and a third opener would create a new one diff --git a/tests/v1/worker/test_gpu_worker_weight_transfer.py b/tests/v1/worker/test_gpu_worker_weight_transfer.py index e576ebd031..537653bda3 100644 --- a/tests/v1/worker/test_gpu_worker_weight_transfer.py +++ b/tests/v1/worker/test_gpu_worker_weight_transfer.py @@ -11,7 +11,7 @@ import pytest import torch import torch.nn as nn -from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config import ParallelConfig, VllmConfig, get_current_vllm_config from vllm.lora.layers import BaseLayerWithLoRA from vllm.v1.worker.gpu_model_runner import _get_parameter_for_reload from vllm.v1.worker.gpu_worker import Worker @@ -136,12 +136,22 @@ def test_rank_local_update_selects_worker_payload(rank, expected): assert worker._weight_update_active is True -def test_rank_local_update_includes_data_parallel_rank(): +def test_rank_local_update_uses_data_parallel_index_after_reconfigure(): engine = _RecordingEngine() worker = _make_worker(engine) worker.rank = 0 - worker.vllm_config.parallel_config.data_parallel_size = 4 - worker.vllm_config.parallel_config.data_parallel_rank = 2 + parallel_config = ParallelConfig( + data_parallel_size=4, + data_parallel_rank=2, + ) + assert parallel_config.data_parallel_rank == 2 + assert parallel_config.data_parallel_index == 2 + + parallel_config.reconfigure_for_independent_dp_rank() + assert parallel_config.data_parallel_rank == 0 + assert parallel_config.data_parallel_index == 2 + + worker.vllm_config.parallel_config = parallel_config Worker.start_weight_update(worker) Worker.update_weights( diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/common.py b/vllm/distributed/ec_transfer/ec_connector/cpu/common.py index e2c37b72a6..eff16e08b0 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/common.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/common.py @@ -99,8 +99,8 @@ def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion: ec_config = vllm_config.ec_transfer_config assert ec_config is not None, "ec_transfer_config required to build region" - dp_rank = vllm_config.parallel_config.data_parallel_rank - engine_id = f"{vllm_config.instance_id}_dp{dp_rank}" + dp_index = vllm_config.parallel_config.data_parallel_index + engine_id = f"{vllm_config.instance_id}_dp{dp_index}" dtype = vllm_config.model_config.dtype hidden_dim = _get_encoder_cache_hidden_dim(vllm_config) diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index a288a97cc1..02f599e57e 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -1449,7 +1449,7 @@ class Worker(WorkerBase): if isinstance(update_info, list): parallel_config = self.vllm_config.parallel_config local_update_info = update_info[ - parallel_config.data_parallel_rank * parallel_config.world_size + parallel_config.data_parallel_index * parallel_config.world_size + self.rank ] else: From 438434b5b5f8936a4dc7d651c708928c90f085d6 Mon Sep 17 00:00:00 2001 From: akii96 Date: Thu, 17 Sep 2026 17:44:54 +0300 Subject: [PATCH 034/143] [ROCm][Perf] Insert MiniMax-M3 sparse-PA K/V without a contiguous copy (#56849) Signed-off-by: Aakif Nawaz --- tests/kernels/test_minimax_m3_amd_ops.py | 76 ++++++++++++++++++++++++ vllm/models/minimax_m3/amd/model.py | 30 +++++++++- 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/tests/kernels/test_minimax_m3_amd_ops.py b/tests/kernels/test_minimax_m3_amd_ops.py index 997c619809..27bd74ce41 100644 --- a/tests/kernels/test_minimax_m3_amd_ops.py +++ b/tests/kernels/test_minimax_m3_amd_ops.py @@ -10,6 +10,7 @@ tests assert the two agree within tolerance: * Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch * Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul * Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math + * Strided K/V sparse-PA cache insert -> contiguous-copy insert The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T`` @@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa: _mxfp8_e4m3_quantize_triton, dequant_mxfp8_to_bf16, ) +from vllm.models.minimax_m3.amd.model import _kv_insert_operand # noqa: E402 from vllm.models.minimax_m3.amd.ops import ( # noqa: E402 gemma_fused_add_rmsnorm, gemma_rmsnorm, @@ -608,3 +610,77 @@ def test_routed_experts_expert_map_delegates_to_kernel(): assert torch.equal(resolve(True), mask) # AITER kernel -> 0/1 mask assert torch.equal(resolve(False), canonical) # non-AITER -> canonical map assert torch.equal(resolve(False, has_moe_kernel=False), canonical) # non-modular + + +# --------------------------------------------------------------------------- # +# Strided K/V operand for the AITER sparse-PA cache insert +# --------------------------------------------------------------------------- # +def _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim, num_q_heads=16): + """K/V as the sparse-PA insert sees them: column slices of a fused qkv.""" + kv = num_kv_heads * head_dim + row = num_q_heads * head_dim + 2 * kv + 2 * head_dim + qkv = torch.randn((num_tokens, row), dtype=torch.bfloat16, device=DEVICE) + k_start = num_q_heads * head_dim + k = qkv[:, k_start : k_start + kv].view(num_tokens, num_kv_heads, head_dim) + v = qkv[:, k_start + kv : k_start + 2 * kv].view(num_tokens, num_kv_heads, head_dim) + return k, v + + +def _asm_kv_cache(pages, num_kv_heads, head_dim): + """Page-16 K/V cache views matching ``_ensure_aiter_sparse_pa_kv_cache``.""" + x = 16 // torch.tensor([], dtype=torch.bfloat16).element_size() + key = torch.zeros( + (pages, num_kv_heads, head_dim // x, 16, x), + dtype=torch.bfloat16, + device=DEVICE, + ) + value = torch.zeros( + (pages, num_kv_heads, 16 // x, head_dim, x), + dtype=torch.bfloat16, + device=DEVICE, + ) + return key, value + + +@pytest.mark.parametrize("num_tokens", [1, 8, 129, 1024]) +@pytest.mark.parametrize("num_kv_heads", [1, 2]) +def test_kv_insert_operand_matches_contiguous(num_tokens, num_kv_heads): + """A strided K/V slice must insert bit-identically to a contiguous copy.""" + reshape_and_cache = pytest.importorskip("aiter").reshape_and_cache + head_dim = 128 + pages = num_tokens // 16 + 4 + slots = torch.randperm(pages * 16, device=DEVICE)[:num_tokens].to(torch.int32) + + torch.manual_seed(num_tokens) + k, v = _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim) + assert num_tokens == 1 or not k.is_contiguous() + + caches = [] + for operand in (lambda t: t.contiguous(), _kv_insert_operand): + key_cache, value_cache = _asm_kv_cache(pages, num_kv_heads, head_dim) + reshape_and_cache( + operand(k), + operand(v), + key_cache, + value_cache, + slots, + kv_cache_dtype="auto", + asm_layout=True, + ) + caches.append((key_cache, value_cache)) + + (ref_k, ref_v), (got_k, got_v) = caches + assert torch.equal(ref_k, got_k) + assert torch.equal(ref_v, got_v) + + +def test_kv_insert_operand_copies_unsupported_layouts(): + """Only a row-strided slice passes through; anything else is copied.""" + k, _ = _fused_qkv_kv_slices(32, 2, 128) + assert _kv_insert_operand(k) is k + + inner_strided = k[:, :, ::2] + assert _kv_insert_operand(inner_strided).is_contiguous() + + flat = k.reshape(32, -1) + assert _kv_insert_operand(flat).is_contiguous() diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 50cdb55183..a0a58253d1 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -579,6 +579,32 @@ class MiniMaxM3Attention(nn.Module): return output +# Widest row pitch aiter's reshape_and_cache can carry in its `int` stride args. +_MAX_KV_INSERT_ROW_STRIDE = 2**31 - 1 + + +def _kv_insert_operand(t: torch.Tensor) -> torch.Tensor: + """K or V operand for ``aiter.reshape_and_cache``, without a needless copy. + + The AITER sparse-PA insert takes K and V as column slices of the fused + ``[q | k | v | index_q | index_k]`` projection, so they are row-strided + views and never contiguous. ``reshape_and_cache`` does not need them to be: + it passes ``key.stride(0)`` to the kernel as the row pitch and reads + ``key[token_idx * key_stride + i]`` for ``i`` over ``num_heads * + head_size``, so the only requirement is that the trailing + ``(num_heads, head_size)`` block be contiguous within a row, which such a + slice always satisfies. Any other layout falls back to a copy. + """ + if ( + t.dim() == 3 + and t.stride(2) == 1 + and t.stride(1) == t.shape[2] + and t.stride(0) <= _MAX_KV_INSERT_ROW_STRIDE + ): + return t + return t.contiguous() + + class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): """Block-sparse attention layer with the lightning-indexer branch. @@ -902,8 +928,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): else "auto" ) reshape_and_cache( - k.contiguous(), - v.contiguous(), + _kv_insert_operand(k), + _kv_insert_operand(v), key_cache, value_cache, slot_mapping, From 3affd35f3164ba17503ba63cd8733becd1b03bf1 Mon Sep 17 00:00:00 2001 From: vllmellm Date: Thu, 17 Sep 2026 22:52:12 +0800 Subject: [PATCH 035/143] [Fix][ROCm] MXFP4 MoE round-up inflates TP-sharded expert weights on CDNA3, starving KV cache (#56359) Signed-off-by: vllmellm Co-authored-by: TJian --- .../layers/fused_moe/oracle/mxfp4.py | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index ce19ba39bf..d1c151aca2 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -769,18 +769,31 @@ def mxfp4_round_up_hidden_size_and_intermediate_size( intermediate_size = round_up(intermediate_size, 128) hidden_size = round_up(hidden_size, 128) elif current_platform.is_rocm(): - if backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and ( - activation == MoEActivation.SITU or activation == MoEActivation.SILU - ): - # K3's AITER A16W4 SiTU kernel handles K3's native intermediate size - # (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 (a - # no-op for K3's shapes) rather than the generic ROCm 256 round-up, - # which would inflate weights and OOM. - intermediate_size = round_up(intermediate_size, 128) - hidden_size = round_up(hidden_size, 128) - else: - intermediate_size = round_up(intermediate_size, 256) - hidden_size = round_up(hidden_size, 256) + from vllm.platforms.rocm import get_cdna_version + + is_situ_or_silu = activation in ( + MoEActivation.SITU, + MoEActivation.SILU, + ) + + # K3's AITER A16W4 SiTU kernel handles K3's native intermediate size + # (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 + # rather than the generic ROCm 256 round-up, which would inflate + # weights and OOM. + aiter_uses_128 = backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 + + # matmul_ogs uses block_k=128 for MXFP4 on pre-CDNA4 GPUs. + # CDNA4's F16xMXFP4 configuration uses block_k=256. + triton_uses_128 = ( + backend == Mxfp4MoeBackend.TRITON_UNFUSED and get_cdna_version() != 4 + ) + + alignment = ( + 128 if is_situ_or_silu and (aiter_uses_128 or triton_uses_128) else 256 + ) + + intermediate_size = round_up(intermediate_size, alignment) + hidden_size = round_up(hidden_size, alignment) elif backend == Mxfp4MoeBackend.CPU: # CPU AMX kernel uses BLOCK_N=32, align to 32 intermediate_size = round_up(intermediate_size, 32) From 4f9a41ac94beb940249d9712a6dd2996cf37da61 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Thu, 17 Sep 2026 11:05:31 -0400 Subject: [PATCH 036/143] [CI/Build][Hardware][NVIDIA] Add Rubin CUDA 13.4 nightly images (#55953) Signed-off-by: Tyler Michael Smith Co-authored-by: OpenAI Codex Co-authored-by: OpenAI Codex Co-authored-by: Misha Goin Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .buildkite/release-pipeline.yaml | 92 +++++++++++++++++++ .buildkite/scripts/cleanup-nightly-builds.sh | 4 +- docker/Dockerfile | 5 + tests/v1/engine/test_output_processor.py | 3 +- .../kv_connector/unit/test_nixl_connector.py | 3 +- vllm/models/qwen4_exp/nvidia/ops/qsa.py | 3 +- 6 files changed, 105 insertions(+), 5 deletions(-) diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 4af4eaf56e..2b90fef820 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -316,6 +316,66 @@ steps: - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" release-images' + - label: "Build release image - x86_64 - Rubin CUDA 13.4" + depends_on: ~ + id: build-release-image-x86-rubin-cuda-13-4 + if: build.env("NIGHTLY") == "1" + agents: + queue: cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker buildx build --load \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \ + --platform linux/amd64 \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=13.4 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg INSTALL_RUBIN_PRERELEASE=true \ + --build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \ + --build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \ + --build-arg RUN_WHEEL_CHECK=false \ + --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.4 \ + --build-arg FINAL_BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.08-cuda13.4-devel-ubuntu24.04 \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images' + + - label: "Build release image - aarch64 - Rubin CUDA 13.4" + depends_on: ~ + id: build-release-image-arm64-rubin-cuda-13-4 + if: build.env("NIGHTLY") == "1" + agents: + queue: arm64_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker buildx build --load \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \ + --platform linux/arm64 \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=13.4 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg INSTALL_RUBIN_PRERELEASE=true \ + --build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \ + --build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \ + --build-arg RUN_WHEEL_CHECK=false \ + --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.4 \ + --build-arg FINAL_BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.08-cuda13.4-devel-ubuntu24.04 \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images' + - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04" depends_on: ~ id: build-release-image-x86-ubuntu2404 @@ -513,6 +573,20 @@ steps: - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" release-manifests' + - label: "Create multi-arch manifest - Rubin CUDA 13.4" + depends_on: + - build-release-image-x86-rubin-cuda-13-4 + - build-release-image-arm64-rubin-cuda-13-4 + id: create-multi-arch-manifest-rubin-cuda-13-4 + if: build.env("NIGHTLY") == "1" + agents: + queue: small_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu134 --amend" + - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: Rubin CUDA 13.4" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" release-manifests' + - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04" depends_on: - build-release-image-x86-ubuntu2404 @@ -585,6 +659,24 @@ steps: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" + - label: "Publish nightly multi-arch Rubin image to DockerHub - CUDA 13.4" + depends_on: + - create-multi-arch-manifest-rubin-cuda-13-4 + if: build.env("NIGHTLY") == "1" + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/push-nightly-builds.sh cu134" + # Clean up old nightly builds (keep only last 14) + - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu134-nightly-" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" + - label: "Publish nightly CPU multi-arch image to DockerHub" depends_on: - build-cpu-release-image-x86 diff --git a/.buildkite/scripts/cleanup-nightly-builds.sh b/.buildkite/scripts/cleanup-nightly-builds.sh index b324778b1f..41b04996e4 100755 --- a/.buildkite/scripts/cleanup-nightly-builds.sh +++ b/.buildkite/scripts/cleanup-nightly-builds.sh @@ -56,8 +56,8 @@ get_all_tags() { set -x # Get both last_updated timestamp and tag name, separated by | - # Exclude nightly-dev tags from cleanup - local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"') + # Exclude mutable architecture tags and nightly-dev tags from cleanup + local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"') if [ -z "$tags" ]; then break diff --git a/docker/Dockerfile b/docker/Dockerfile index d1a7afcbfb..ad010285b1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -231,10 +231,12 @@ RUN --mount=type=cache,target=/opt/uv/cache \ echo "Installing Rubin prerelease requirements..." \ && /opt/venv/bin/python3 use_existing_torch.py --prefix \ && uv pip install --python /opt/venv/bin/python3 \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt \ && uv pip install --python /opt/venv/bin/python3 \ -r requirements/cuda.txt \ && uv pip install --python /opt/venv/bin/python3 \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt; \ elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ echo "Installing torch nightly..." \ @@ -941,11 +943,13 @@ RUN --mount=type=cache,target=/opt/uv/cache \ fi && \ if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ uv pip install --system \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt \ && sed -Ei '/^[[:space:]]*(torch|torchvision|torchaudio)[[:space:]]*==/Id' \ /tmp/requirements-cuda.txt \ && uv pip install --system -r /tmp/requirements-cuda.txt \ && uv pip install --system \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt \ && printf '%s\n' \ "/usr/local/lib/python${PYTHON_VERSION}/dist-packages/nvpl/lib" \ @@ -1262,6 +1266,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ echo "Installing Rubin prerelease requirements..." \ && uv pip install --system \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt; \ fi diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 9df43088e6..69128f55b7 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -177,7 +177,8 @@ def test_incremental_detokenization( def test_remote_prefill_cached_tokens_override(do_remote_prefill: bool): """P/D disaggregation: num_cached_tokens should report the P worker's cache hits (passed via kv_transfer_params) instead of the local count, - which sees the KVs pulled from the remote prefill as a ~100% hit.""" + which sees the KVs pulled from the remote prefill as a ~100% hit. + """ output_processor = OutputProcessor(tokenizer=None, log_stats=False) prompt_tokens = [1, 2, 3, 4, 5, 6, 7, 8] diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 2185ba62d2..9ab4b0ae95 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -350,7 +350,8 @@ def test_prefill_exports_cached_tokens_in_kv_transfer_params(): """The P worker reports its own prefix-cache hits in the returned kv_transfer_params so the D worker can surface them in prompt_tokens_details instead of the ~100% local hit it measures - when pulling the KVs from the remote.""" + when pulling the KVs from the remote. + """ vllm_config = create_vllm_config() scheduler = create_scheduler(vllm_config) diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa.py b/vllm/models/qwen4_exp/nvidia/ops/qsa.py index 816ccdd498..401048e02c 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa.py @@ -482,7 +482,8 @@ def _select_sm120_config( """(block_n, target_splits, num_warps) retuned on sm_120 (RTX PRO 6000 Blackwell), split by cache dtype: bf16 and fp8 favour different configs on sm_120, most visibly on the large-prefill region. Each entry is the fastest - config that stays correct on its own path; the main win is more warps.""" + config that stays correct on its own path; the main win is more warps. + """ if is_fp8: if base_programs > 2048: return (32, 2, 1) if use_prefill_config else (64, 1, 2) From 1df336c3bab8e2ac2eb3d325bd90e62b94656e9c Mon Sep 17 00:00:00 2001 From: Mikko Tukiainen Date: Thu, 17 Sep 2026 18:05:48 +0300 Subject: [PATCH 037/143] [Perf][ROCm] Enable AITER QuickReduce + RMSNorm fusion (#48249) Signed-off-by: Mikko Tukiainen Co-authored-by: Cursor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/distributed/test_quick_all_reduce.py | 172 +++++++++++++++++++++ vllm/_aiter_ops.py | 107 ++++++++++++- 2 files changed, 278 insertions(+), 1 deletion(-) diff --git a/tests/distributed/test_quick_all_reduce.py b/tests/distributed/test_quick_all_reduce.py index f9228fbd68..107d95fa61 100644 --- a/tests/distributed/test_quick_all_reduce.py +++ b/tests/distributed/test_quick_all_reduce.py @@ -3,12 +3,17 @@ import multiprocessing import random +import sys +import types +from types import SimpleNamespace import pytest import ray import torch import torch.distributed as dist +import vllm._aiter_ops as aiter_ops +import vllm.distributed as vllm_distributed import vllm.envs as envs from vllm import _custom_ops as ops from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa @@ -233,6 +238,173 @@ def test_quick_allreduce_passes_dynamic_quant_level( assert called_quant_level == QuickReduceRegime.FP.value +def test_rocm_aiter_fused_rmsnorm_uses_aiter_qr_rmsnorm_for_prefill( + monkeypatch: pytest.MonkeyPatch, +): + class FakeAiterAllReduce: + aiter_ca = SimpleNamespace(world_size=2, fully_connected=True) + + def use_1stage_fused_ar_rms(self, inp): + return False + + def custom_fused_ar_rms(self, *args, **kwargs): + raise AssertionError("expected fused QR+RMSNorm dispatch") + + class FakeQuickReduce: + disabled = False + + def __init__(self): + self.checked_input = None + + def should_quick_allreduce(self, inp): + self.checked_input = inp + return True + + def _get_qr_quant_level(self, inp): + return QuickReduceRegime.INT4.value + + qr_comm = FakeQuickReduce() + device_comm = SimpleNamespace(cpu_group=object(), qr_comm=qr_comm) + monkeypatch.setattr( + vllm_distributed, + "get_tp_group", + lambda: SimpleNamespace(device_communicator=device_comm), + ) + monkeypatch.setattr( + aiter_ops.rocm_aiter_ops, + "get_aiter_allreduce", + lambda: FakeAiterAllReduce(), + ) + monkeypatch.setattr(aiter_ops.dist, "get_world_size", lambda group: 2) + monkeypatch.setattr(aiter_ops.dist, "get_rank", lambda group: 0) + monkeypatch.setattr( + aiter_ops.dist, + "all_gather_object", + lambda handles, handle, group: handles.__setitem__(0, handle), + ) + + calls = [] + fake_aiter = types.SimpleNamespace( + init_custom_qr=lambda rank, world_size, qr_max_size: 123, + qr_get_handle=lambda ptr: b"handle", + qr_open_handles=lambda ptr, handles: None, + ) + + def fake_qr_all_reduce_rmsnorm( + ptr, + inp, + residual, + residual_out, + out, + weight, + eps, + hidden_dim, + quant_level, + cast_bf2half, + ): + calls.append( + { + "ptr": ptr, + "inp": inp, + "residual": residual, + "residual_out": residual_out, + "out": out, + "weight": weight, + "eps": eps, + "hidden_dim": hidden_dim, + "quant_level": quant_level, + "cast_bf2half": cast_bf2half, + } + ) + out.copy_(inp) + residual_out.copy_(residual) + + fake_aiter.qr_all_reduce_rmsnorm = fake_qr_all_reduce_rmsnorm + monkeypatch.setitem(sys.modules, "aiter", fake_aiter) + monkeypatch.setattr(envs, "VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", None) + monkeypatch.setattr(envs, "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", True) + + # token_num > 80 forces vLLM's AITER fused AR+RMS wrapper onto its + # non-1stage prefill path. hidden_dim=4096 bf16 gives an 8 KiB row, so four + # independent RMSNorm rows fit in one 32 KiB QR tile. + inp = torch.randn(81, 4096, dtype=torch.bfloat16) + residual = torch.randn_like(inp) + weight = torch.randn(4096, dtype=torch.bfloat16) + + out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl( + inp, residual, weight, 1e-6 + ) + + assert qr_comm.checked_input is inp + assert len(calls) == 1 + assert calls[0]["ptr"] == 123 + assert calls[0]["inp"] is inp + assert calls[0]["residual"] is residual + assert calls[0]["out"] is out + assert calls[0]["residual_out"] is residual_out + assert calls[0]["weight"] is weight + assert calls[0]["hidden_dim"] == 4096 + assert calls[0]["quant_level"] == QuickReduceRegime.INT4.value + assert calls[0]["cast_bf2half"] is True + torch.testing.assert_close(out, inp) + torch.testing.assert_close(residual_out, residual) + + +def test_rocm_aiter_fused_rmsnorm_keeps_1stage_decode_path( + monkeypatch: pytest.MonkeyPatch, +): + calls = [] + + class FakeAiterCA: + world_size = 2 + fully_connected = True + + def custom_fused_ar_rms( + self, inp, residual, weight, epsilon, *, use_1stage, gemma_norm + ): + calls.append( + { + "inp": inp, + "residual": residual, + "weight": weight, + "eps": epsilon, + "use_1stage": use_1stage, + "gemma_norm": gemma_norm, + } + ) + return inp + 1, residual + 1 + + class FakeAiterAllReduce: + aiter_ca = FakeAiterCA() + + def use_1stage_fused_ar_rms(self, inp): + return True + + monkeypatch.setattr( + aiter_ops.rocm_aiter_ops, + "get_aiter_allreduce", + lambda: FakeAiterAllReduce(), + ) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + + inp = torch.randn(4, 4096, dtype=torch.bfloat16) + residual = torch.randn_like(inp) + weight = torch.randn(4096, dtype=torch.bfloat16) + + out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl( + inp, residual, weight, 1e-6 + ) + + assert len(calls) == 1 + assert calls[0]["inp"] is inp + assert calls[0]["residual"] is residual + assert calls[0]["weight"] is weight + assert calls[0]["use_1stage"] is True + assert calls[0]["gemma_norm"] is False + torch.testing.assert_close(out, inp + 1) + torch.testing.assert_close(residual_out, residual + 1) + + @ray.remote(num_gpus=1, max_calls=1) def graph_quickreduce( monkeypatch: pytest.MonkeyPatch, diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 101e1dd10a..4b9a145f48 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -4,9 +4,10 @@ import ctypes import functools import os from collections.abc import Callable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import torch +import torch.distributed as dist from torch._ops import OpOverload import vllm.envs as envs @@ -34,6 +35,63 @@ except ImportError: # which is a host op, so we cache it once here. FP8_DTYPE = current_platform.fp8_dtype() _HIPB_MM_INITIALIZED_DEVICES: set[int] = set() +KB = 1024 +MB = 1024 * KB + + +def _get_or_create_aiter_qr_rmsnorm_comm( + device_comm: Any, +) -> tuple[int, int, bool] | None: + """Create the AITER QR communicator used by fused QR+RMSNorm. + + vLLM's normal QuickReduce communicator owns vLLM C++ QR buffers. The fused + QR+RMSNorm kernel added in AITER needs an AITER DeviceComms instance, so we + lazily create one on the same TP CPU group and keep it on the vLLM device + communicator for reuse. + """ + existing = getattr(device_comm, "_aiter_qr_rmsnorm_comm", None) + if existing is False: + return None + if existing is not None: + return existing + + try: + import aiter + except Exception: + device_comm._aiter_qr_rmsnorm_comm = False + return None + + if not hasattr(aiter, "qr_all_reduce_rmsnorm"): + device_comm._aiter_qr_rmsnorm_comm = False + return None + + group = getattr(device_comm, "cpu_group", None) + if group is None: + device_comm._aiter_qr_rmsnorm_comm = False + return None + + world_size = dist.get_world_size(group=group) + rank = dist.get_rank(group=group) + if world_size not in (2, 4, 8): + device_comm._aiter_qr_rmsnorm_comm = False + return None + + qr_max_size_mb = envs.VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB + qr_max_size = None if qr_max_size_mb is None else qr_max_size_mb * MB + + try: + ptr = aiter.init_custom_qr(rank, world_size, qr_max_size) + handle = aiter.qr_get_handle(ptr) + handles = [None] * world_size + dist.all_gather_object(handles, handle, group=group) + aiter.qr_open_handles(ptr, handles) + except Exception: + device_comm._aiter_qr_rmsnorm_comm = False + return None + + comm = (ptr, world_size, envs.VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16) + device_comm._aiter_qr_rmsnorm_comm = comm + return comm def _ensure_hipb_mm_extension_initialized() -> None: @@ -991,6 +1049,53 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl( ca = aiter_ar.aiter_ca use_1stage = aiter_ar.use_1stage_fused_ar_rms(input_) + if not use_1stage: + from vllm.distributed import get_tp_group + + device_comm = get_tp_group().device_communicator + qr_comm = getattr(device_comm, "qr_comm", None) + hidden_dim = input_.shape[-1] + row_size = hidden_dim * input_.element_size() + fused_qr_rmsnorm_ok = ( + qr_comm is not None + and not getattr(qr_comm, "disabled", True) + and hasattr(qr_comm, "should_quick_allreduce") + and qr_comm.should_quick_allreduce(input_) + and input_.shape == residual.shape + and input_.dtype in (torch.bfloat16, torch.float16) + and input_.dtype == residual.dtype + and input_.dtype == weight.dtype + and weight.dim() == 1 + and weight.numel() == hidden_dim + and input_.numel() % hidden_dim == 0 + and row_size > 0 + and row_size <= 32 * KB + and (32 * KB) % row_size == 0 + ) + if fused_qr_rmsnorm_ok: + assert qr_comm is not None + aiter_qr_comm = _get_or_create_aiter_qr_rmsnorm_comm(device_comm) + if aiter_qr_comm is not None: + import aiter + + ptr, _, cast_bf2half = aiter_qr_comm + quant_level = qr_comm._get_qr_quant_level(input_) + out = torch.empty_like(input_) + residual_out = torch.empty_like(residual) + aiter.qr_all_reduce_rmsnorm( + ptr, + input_, + residual, + residual_out, + out, + weight, + epsilon, + hidden_dim, + quant_level, + cast_bf2half, + ) + return out, residual_out + result = ca.custom_fused_ar_rms( input_, residual, From 493d4b3c04a439c926df2c4d2fa3cdfabb1a8605 Mon Sep 17 00:00:00 2001 From: HieDean <34408026+HieDean@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:20:37 +0800 Subject: [PATCH 038/143] [bugfix] Mark draft tokens to rebuilt their embeddings. (#57356) Signed-off-by: HieDean <799287043@qq.com> Co-authored-by: hiedean --- vllm/v1/worker/gpu_model_runner.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 10f85be718..77d11621c2 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -1858,8 +1858,12 @@ class GPUModelRunner( total_without_spec = total_num_scheduled_tokens - total_num_spec_tokens if self.enable_prompt_embeds: # The multimodal embed path reads is_token_ids.gpu; its .cpu copy is - # refreshed every step but the async fast paths below only scatter - # input_ids.gpu, so refresh is_token_ids.gpu here too. + # refreshed every step but the async fast path below replaces draft + # tokens directly on the GPU. Mark those positions as token IDs before + # refreshing the GPU mask so their embeddings are rebuilt. Sampled + # tokens are already marked by bookkeeping. + if spec_flattened_indices: + self.is_token_ids.np[spec_flattened_indices] = True self.is_token_ids.copy_to_gpu(total_num_scheduled_tokens) if num_common_tokens < total_without_spec: # If not all requests are decodes from the last iteration, From 6a7de3ba3ad95965e41618568ff803fa01ab168d Mon Sep 17 00:00:00 2001 From: Zoe923 <52874215+Zoe923@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:49:56 +0800 Subject: [PATCH 039/143] [Bugfix][GDN] Fix CuteDSL BF16 KKT inversion divergence (#53864) Signed-off-by: Zoe923 <871844693@qq.com> --- .../kernels/mamba/test_gdn_prefill_cutedsl.py | 83 +++++++++++++++++++ .../gdn_chunk_cutedsl/kernel_kkt_inv_uw.py | 4 +- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/tests/kernels/mamba/test_gdn_prefill_cutedsl.py b/tests/kernels/mamba/test_gdn_prefill_cutedsl.py index 861b686e32..d86cfde2c7 100644 --- a/tests/kernels/mamba/test_gdn_prefill_cutedsl.py +++ b/tests/kernels/mamba/test_gdn_prefill_cutedsl.py @@ -192,3 +192,86 @@ def test_gdn_chunk_cutedsl_correctness(num_seqs: int, state_dtype: torch.dtype): assert no_buffer_state_error.mean().item() < 6e-4 assert buffer_o_error.max().item() == 0 assert buffer_state_error.max().item() == 0 + + +def test_gdn_chunk_cutedsl_correlated_keys_stability(): + """Cover a KKT block where three BF16 Newton rounds diverge.""" + total_tokens = 64 + num_k_heads = 4 + num_v_heads = 8 + head_dim = 128 + rng = torch.Generator("cuda").manual_seed(3456) + + q = torch.zeros( + 1, + total_tokens, + num_k_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + k = torch.zeros_like(q) + q[..., 0] = 1 + k[..., 0] = 1 + v = torch.randn( + 1, + total_tokens, + num_v_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + generator=rng, + ) + g = torch.zeros( + 1, + total_tokens, + num_v_heads, + device="cuda", + dtype=torch.float32, + ) + beta = torch.full_like(g, 0.9) + initial_state = torch.zeros( + 1, + num_v_heads, + head_dim, + head_dim, + device="cuda", + dtype=torch.float32, + ) + cu_seqlens = torch.tensor([0, total_tokens], device="cuda", dtype=torch.int32) + chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens) + + ref_o, ref_state = chunk_gated_delta_rule( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + ) + actual_o, actual_state = chunk_gated_delta_rule_cutedsl( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + ) + torch.accelerator.synchronize() + + assert torch.isfinite(actual_o).all() + assert torch.isfinite(actual_state).all() + output_error = (actual_o.float() - ref_o.float()).abs() + state_error = ( + actual_state.float() - ref_state.to(actual_state.dtype).float() + ).abs() + assert output_error.max().item() < 1e-1 + assert output_error.mean().item() < 2e-3 + assert state_error.max().item() < 2e-1 + assert state_error.mean().item() < 6e-4 diff --git a/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py b/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py index ba61ec68b5..4b4046a093 100644 --- a/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py +++ b/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py @@ -573,8 +573,8 @@ class Sm100ChunkUWKernel: for i in cutlass.range_constexpr(4): M[i] = _bf16x2_sub(_bf16x2_neg(eye[i]), M[i]) - # 3 rounds of Newton-Schulz - for _ in cutlass.range_constexpr(3): + # The fourth round keeps correlated-key KKT blocks stable in BF16. + for _ in cutlass.range_constexpr(4): # First MMA: -AiM = Ai @ (-M) cute.copy(stsm_atom, Ai_bf16, sA_ldsm[None, (warp_id_, warp_id_)]) cute.arch.sync_warp() From d7e755c23080988f7450f107466d6f6a74b042ff Mon Sep 17 00:00:00 2001 From: Ziyang Ma Date: Thu, 17 Sep 2026 23:50:14 +0800 Subject: [PATCH 040/143] [Bugfix] Fix wrong vLLM version reported by pip install (proto-v* tag collision) (#57295) Signed-off-by: RyanMa29 Signed-off-by: Bugen Zhao Co-authored-by: Bugen Zhao Co-authored-by: Codex --- .buildkite/scripts/ci-bake-rocm.sh | 2 +- setup.py | 22 ++++++++++++++++++++-- tools/build_rust.py | 17 ++++++++++++++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 6c5f3308b7..0a9e6e7589 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -418,7 +418,7 @@ validate_ci_build_context_source() { describe_ci_revision() { git -C "$1" describe --tags --long --abbrev=10 \ - --match '*[0-9]*' "$2" 2>/dev/null + --match 'v[0-9]*' "$2" 2>/dev/null } write_ci_git_archival_metadata() { diff --git a/setup.py b/setup.py index b24bcc1a2c..ed9616195f 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,18 @@ from setuptools_rust.build import build_rust from setuptools_scm import get_version from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME +# Select vLLM release tags, excluding crate tags such as "proto-v0.3.0". +VLLM_GIT_DESCRIBE_COMMAND = [ + "git", + "describe", + "--dirty", + "--tags", + "--long", + "--abbrev=40", + "--match", + "v[0-9]*", +] + def load_module_from_path(module_name, path): spec = importlib.util.spec_from_file_location(module_name, path) @@ -1249,9 +1261,15 @@ def get_vllm_version() -> str: if env_version := os.getenv("VLLM_VERSION_OVERRIDE"): print(f"Overriding VLLM version with {env_version} from VLLM_VERSION_OVERRIDE") os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = env_version - return get_version(write_to="vllm/_version.py") + return get_version( + write_to="vllm/_version.py", + git_describe_command=VLLM_GIT_DESCRIBE_COMMAND, + ) - version = get_version(write_to="vllm/_version.py") + version = get_version( + write_to="vllm/_version.py", + git_describe_command=VLLM_GIT_DESCRIBE_COMMAND, + ) sep = "+" if "+" not in version else "." # dev versions might contain + if not envs.VLLM_SKIP_VERSION_SUFFIX: diff --git a/tools/build_rust.py b/tools/build_rust.py index a000ec8169..320cf75e3d 100644 --- a/tools/build_rust.py +++ b/tools/build_rust.py @@ -16,13 +16,28 @@ from setuptools_scm import get_version ROOT_DIR = Path(__file__).resolve().parents[1] VLLM_RS_BUILD_VERSION = "VLLM_RS_BUILD_VERSION" +# Select vLLM release tags, excluding crate tags such as "proto-v0.3.0". +VLLM_GIT_DESCRIBE_COMMAND = [ + "git", + "describe", + "--dirty", + "--tags", + "--long", + "--abbrev=40", + "--match", + "v[0-9]*", +] + def prepare_build_environment() -> str | None: """Set the device-independent vLLM source version for Rust artifacts.""" version = os.getenv(VLLM_RS_BUILD_VERSION) or None if version is None: try: - version = get_version(root=ROOT_DIR) + version = get_version( + root=ROOT_DIR, + git_describe_command=VLLM_GIT_DESCRIBE_COMMAND, + ) except LookupError: return None From b78fbab4be552650286778a9e489980d850f76c3 Mon Sep 17 00:00:00 2001 From: Young <33064860+YCH188@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:52:10 +0800 Subject: [PATCH 041/143] [Docs] Fix the typos in the document (#57190) Co-authored-by: Young --- docs/benchmarking/cli.md | 4 ++-- docs/benchmarking/dashboard.md | 2 +- docs/contributing/README.md | 2 +- docs/design/metrics.md | 2 +- docs/features/quantization/quark.md | 2 +- docs/features/speculative_decoding/draft_model.md | 2 +- docs/getting_started/installation/cpu.x86.inc.md | 2 +- docs/governance/committers.md | 2 +- docs/training/weight_transfer/base.md | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index d46ab04b0e..e7ee794da6 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -147,7 +147,7 @@ With standard decoding, each streamed output usually contains one token, so ITL and TPOT are typically similar. With speculative decoding, one streamed output can contain multiple tokens, -such as several accepted draft tokens within a single engine tstep. ITL records +such as several accepted draft tokens within a single engine step. ITL records only the gaps between streamed outputs; it does not add zero-duration gaps for tokens in the same output. TPOT instead amortizes the request's decoding time over every output token. @@ -451,7 +451,7 @@ vllm bench serve \ --num-prompts -1 ``` -Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructued data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card. +Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructured data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card. #### BFCL (Tool-Calling) Benchmark diff --git a/docs/benchmarking/dashboard.md b/docs/benchmarking/dashboard.md index 44effc078e..4c1132c17f 100644 --- a/docs/benchmarking/dashboard.md +++ b/docs/benchmarking/dashboard.md @@ -36,7 +36,7 @@ When run, benchmark script generates results under **benchmark/results** folder, - `ON_CPU`: set the value to '1' on Intel® Xeon® and Arm® Neoverse™ Processors. Default value is 0. - `SERVING_JSON`: JSON file to use for the serving tests. Default value is empty string (use default file). - `LATENCY_JSON`: JSON file to use for the latency tests. Default value is empty string (use default file). -- `THROUGHPUT_JSON`: JSON file to use for the throughout tests. Default value is empty string (use default file). +- `THROUGHPUT_JSON`: JSON file to use for the throughput tests. Default value is empty string (use default file). - `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string. - `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string. - `PROMPTS_PER_CONCURRENCY`: Multiplier to compute `num_prompts` for serving tests (`num_prompts = max_concurrency × value`). Overrides JSON `num_prompts`. Default is NULL. diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 7a1af9c1a7..e4b86f99ae 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -50,7 +50,7 @@ To rebuild only the Rust frontend binary: ./build_rust.sh --debug # faster build for development ``` -If you are developing vLLM's Python and CUDA/C++ code, install Pytorch first: +If you are developing vLLM's Python and CUDA/C++ code, install PyTorch first: ```bash uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu129 diff --git a/docs/design/metrics.md b/docs/design/metrics.md index 3d108890d7..3e070f5888 100644 --- a/docs/design/metrics.md +++ b/docs/design/metrics.md @@ -670,7 +670,7 @@ fall under the more general heading of "Observability". vLLM has support for OpenTelemetry tracing: - Added by and reinstated by -- Configured with `--oltp-traces-endpoint` and `--collect-detailed-traces` +- Configured with `--otlp-traces-endpoint` and `--collect-detailed-traces` - [OpenTelemetry blog post](https://opentelemetry.io/blog/2024/llm-observability/) - [User-facing docs](../../examples/observability/opentelemetry/README.md) - [Blog post](https://medium.com/@ronen.schaffer/follow-the-trail-supercharging-vllm-with-opentelemetry-distributed-tracing-aa655229b46f) diff --git a/docs/features/quantization/quark.md b/docs/features/quantization/quark.md index 8445288ff9..e5cfb8653b 100644 --- a/docs/features/quantization/quark.md +++ b/docs/features/quantization/quark.md @@ -100,7 +100,7 @@ kv-cache and the quantization algorithm is AutoSmoothQuant. !!! note Note the quantization algorithm needs a JSON config file and the config file is located in - [Quark Pytorch examples](https://quark.docs.amd.com/latest/pytorch/pytorch_examples.html), + [Quark PyTorch examples](https://quark.docs.amd.com/latest/pytorch/pytorch_examples.html), under the directory `examples/torch/language_modeling/llm_ptq/models`. For example, AutoSmoothQuant config file for Llama is `examples/torch/language_modeling/llm_ptq/models/llama/autosmoothquant_config.json`. diff --git a/docs/features/speculative_decoding/draft_model.md b/docs/features/speculative_decoding/draft_model.md index 5ab4629409..880ea1df59 100644 --- a/docs/features/speculative_decoding/draft_model.md +++ b/docs/features/speculative_decoding/draft_model.md @@ -96,7 +96,7 @@ The code used to request completions as a client remains unchanged: }, gpu_memory_utilization=0.5, ) -outputs = llm.generate(prompts,sampling_params) +outputs = llm.generate(prompts, sampling_params) for output in outputs: prompt = output.prompt diff --git a/docs/getting_started/installation/cpu.x86.inc.md b/docs/getting_started/installation/cpu.x86.inc.md index a64d0e5703..260ffe2863 100644 --- a/docs/getting_started/installation/cpu.x86.inc.md +++ b/docs/getting_started/installation/cpu.x86.inc.md @@ -33,7 +33,7 @@ uv pip install "https://github.com/vllm-project/vllm/releases/download/v${VLLM_V pip install "https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cpu-cp38-abi3-manylinux_2_34_x86_64.whl" --extra-index-url https://download.pytorch.org/whl/cpu ``` !!! warning "set `LD_PRELOAD`" - Before use vLLM CPU installed via wheels, make Intel OpenMP is added to `LD_PRELOAD`: + Before use vLLM CPU installed via wheels, make sure Intel OpenMP is added to `LD_PRELOAD`: ```bash # manually find the path sudo find / -iname *libiomp5.so diff --git a/docs/governance/committers.md b/docs/governance/committers.md index bd36759122..48ed4b2607 100644 --- a/docs/governance/committers.md +++ b/docs/governance/committers.md @@ -2,7 +2,7 @@ This document lists the current committers of the vLLM project and the core areas they maintain. Committers have write access to the vLLM repository and are responsible for reviewing and merging PRs. -You can also refer to the [CODEOWNERS](https://github.com/vllm-project/vllm/blob/main/.github/CODEOWNERS) file for concrete file-level ownership and reviewers. Both this documents and the CODEOWNERS file are living documents and they complement each other. +You can also refer to the [CODEOWNERS](https://github.com/vllm-project/vllm/blob/main/.github/CODEOWNERS) file for concrete file-level ownership and reviewers. Both these documents and the CODEOWNERS file are living documents and they complement each other. ## Active Committers diff --git a/docs/training/weight_transfer/base.md b/docs/training/weight_transfer/base.md index d73c88436a..759c55ef6f 100644 --- a/docs/training/weight_transfer/base.md +++ b/docs/training/weight_transfer/base.md @@ -233,7 +233,7 @@ Two implementations ship with vLLM: | `RayVLLMWeightSyncClient(handle)` | One or more `AsyncLLM`/`LLM` Ray actors. Accepts a list and fans each call out to every handle, blocking on all of them, so a multi-actor (e.g. multi-DP) deployment is driven as one unit | | `HTTPVLLMWeightSyncClient(base_url, timeout=300)` | A vLLM server over the RLHF HTTP routes | -Custom weight sync clients can be implement like so: +Custom weight sync clients can be implemented like so: ```python class MyFrameworkWeightSyncClient: From e960ead0aa70362a57f205be831083b68c7a5bdb Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Thu, 17 Sep 2026 11:11:31 -0500 Subject: [PATCH 042/143] [Mypy] Fix mypy typing for Transformers models (#54320) Signed-off-by: Taneem Ibrahim Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- tests/models/transformers/fusers/test_mla.py | 6 +-- tools/pre_commit/mypy.py | 1 - vllm/compilation/decorators.py | 13 +++-- .../models/transformers/__init__.py | 24 ++++----- .../models/transformers/base.py | 44 +++++++++------ .../models/transformers/causal.py | 4 +- .../models/transformers/fusers/attention.py | 12 +++-- .../models/transformers/fusers/base.py | 7 ++- .../models/transformers/fusers/mla.py | 36 ++++++------- .../models/transformers/fusers/rms_norm.py | 12 ++--- .../models/transformers/fx_utils.py | 25 +++++---- .../models/transformers/legacy.py | 31 ++++++----- .../model_executor/models/transformers/moe.py | 42 +++++++++++---- .../models/transformers/multimodal.py | 54 ++++++++++++------- .../models/transformers/pooling.py | 37 ++++++++----- .../models/transformers/utils.py | 21 ++++---- 16 files changed, 224 insertions(+), 145 deletions(-) diff --git a/tests/models/transformers/fusers/test_mla.py b/tests/models/transformers/fusers/test_mla.py index 9f51ab5662..625fc4dc08 100644 --- a/tests/models/transformers/fusers/test_mla.py +++ b/tests/models/transformers/fusers/test_mla.py @@ -53,7 +53,6 @@ def _match(q_lora_rank: int | None) -> MLAFuser | None: def test_discovers_modules_without_q_lora(): fuser = _match(q_lora_rank=None) assert isinstance(fuser, MLAFuser) - assert not fuser.has_q_lora assert fuser.q_proj_name == "q_proj" assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa" assert fuser.kv_a_layernorm_name == "kv_a_layernorm" @@ -68,7 +67,6 @@ def test_discovers_modules_without_q_lora(): def test_discovers_modules_with_q_lora(): fuser = _match(q_lora_rank=64) assert isinstance(fuser, MLAFuser) - assert fuser.has_q_lora assert fuser.q_a_proj_name == "q_a_proj" assert fuser.q_a_layernorm_name == "q_a_layernorm" assert fuser.q_b_proj_name == "q_b_proj" @@ -107,7 +105,7 @@ def test_update_forward_rewrites_real_attention(q_lora_rank): assert isinstance(fuser, MLAFuser) fuser.update_forward(_attention(q_lora_rank)) names = set(fuser.fused_forward.__code__.co_names) - if fuser.has_q_lora: + if fuser.q_a_proj_name is not None: assert _FUSED_QKV_A_PROJ in names assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names @@ -158,7 +156,7 @@ def test_discovers_modules_under_arbitrary_names(): module = RenamedMLA() fuser = MLAFuser.match(trace(module), module) assert isinstance(fuser, MLAFuser) - assert not fuser.has_q_lora + assert fuser.q_a_proj_name is None assert fuser.q_proj_name == "alpha" assert fuser.kv_a_proj_name == "beta" assert fuser.kv_a_layernorm_name == "gamma" diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index b7a278b9bd..234acc0298 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -116,7 +116,6 @@ EXCLUDE = [ r"vllm/model_executor/models/[nN]", r"vllm/model_executor/models/[oO]", r"vllm/model_executor/models/[qQ]", - r"vllm/model_executor/models/transformers", r"vllm/model_executor/models/[uU]", r"vllm/model_executor/models/[vV]", r"vllm/model_executor/models/[wW]", diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 242e7227e9..35c0e6dee5 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -7,7 +7,7 @@ import inspect import os import sys from collections.abc import Callable, Generator -from typing import TYPE_CHECKING, Any, TypeVar, overload +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, overload from unittest.mock import patch import torch @@ -32,6 +32,9 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer from .monitor import monitor_profiling_run, monitor_torch_compile +DynamicArgDims: TypeAlias = dict[str, int | list[int] | dict[int, str]] +"""Argument name -> the dynamic dimension(s) of that argument.""" + # shape_id parameter was added to mark_unbacked in PyTorch 2.11.0 _SUPPORTS_SHAPE_ID = is_torch_equal_or_newer("2.11.0") @@ -89,7 +92,7 @@ def support_torch_compile( @overload def support_torch_compile( *, - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None, + dynamic_arg_dims: DynamicArgDims | None, ) -> Callable[[type[_T]], type[_T]]: ... @@ -103,7 +106,7 @@ def support_torch_compile( @overload def support_torch_compile( *, - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None, + dynamic_arg_dims: DynamicArgDims | None = None, mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, is_encoder: bool = False, @@ -117,7 +120,7 @@ def support_torch_compile(cls: type[_T]) -> type[_T]: ... def support_torch_compile( cls: type[_T] | None = None, *, - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None, + dynamic_arg_dims: DynamicArgDims | None = None, mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, is_encoder: bool = False, @@ -328,7 +331,7 @@ def _try_load_aot_compiled_fn( def _support_torch_compile( cls: type[_T], - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]], + dynamic_arg_dims: DynamicArgDims, mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, is_encoder: bool = False, diff --git a/vllm/model_executor/models/transformers/__init__.py b/vllm/model_executor/models/transformers/__init__.py index 2d127c430e..e19de6d4e8 100644 --- a/vllm/model_executor/models/transformers/__init__.py +++ b/vllm/model_executor/models/transformers/__init__.py @@ -21,7 +21,7 @@ from typing import TYPE_CHECKING import torch.nn.functional as F from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS -from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR, Base +from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR from vllm.model_executor.models.transformers.causal import CausalMixin from vllm.model_executor.models.transformers.legacy import LegacyMixin from vllm.model_executor.models.transformers.moe import MoEMixin @@ -126,10 +126,10 @@ ALL_ATTENTION_FUNCTIONS.register("vllm_mla", vllm_mla_attention_forward) # Text only models -class TransformersForCausalLM(CausalMixin, Base): ... +class TransformersForCausalLM(CausalMixin): ... -class TransformersMoEForCausalLM(MoEMixin, CausalMixin, Base): ... +class TransformersMoEForCausalLM(MoEMixin, CausalMixin): ... # Multimodal models @@ -138,7 +138,7 @@ class TransformersMoEForCausalLM(MoEMixin, CausalMixin, Base): ... info=MultiModalProcessingInfo, dummy_inputs=MultiModalDummyInputsBuilder, ) -class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin, Base): ... +class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin): ... @MULTIMODAL_REGISTRY.register_processor( @@ -146,16 +146,14 @@ class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin, Base): ... info=MultiModalProcessingInfo, dummy_inputs=MultiModalDummyInputsBuilder, ) -class TransformersMultiModalMoEForCausalLM( - MoEMixin, MultiModalMixin, CausalMixin, Base -): ... +class TransformersMultiModalMoEForCausalLM(MoEMixin, MultiModalMixin, CausalMixin): ... # Embedding models -class TransformersEmbeddingModel(EmbeddingMixin, LegacyMixin, Base): ... +class TransformersEmbeddingModel(EmbeddingMixin, LegacyMixin): ... -class TransformersMoEEmbeddingModel(EmbeddingMixin, MoEMixin, Base): ... +class TransformersMoEEmbeddingModel(EmbeddingMixin, MoEMixin): ... @MULTIMODAL_REGISTRY.register_processor( @@ -163,17 +161,17 @@ class TransformersMoEEmbeddingModel(EmbeddingMixin, MoEMixin, Base): ... info=MultiModalProcessingInfo, dummy_inputs=MultiModalDummyInputsBuilder, ) -class TransformersMultiModalEmbeddingModel(EmbeddingMixin, MultiModalMixin, Base): ... +class TransformersMultiModalEmbeddingModel(EmbeddingMixin, MultiModalMixin): ... # Sequence classification models class TransformersForSequenceClassification( - SequenceClassificationMixin, LegacyMixin, Base + SequenceClassificationMixin, LegacyMixin ): ... class TransformersMoEForSequenceClassification( - SequenceClassificationMixin, MoEMixin, Base + SequenceClassificationMixin, MoEMixin ): ... @@ -183,7 +181,7 @@ class TransformersMoEForSequenceClassification( dummy_inputs=MultiModalDummyInputsBuilder, ) class TransformersMultiModalForSequenceClassification( - SequenceClassificationMixin, MultiModalMixin, Base + SequenceClassificationMixin, MultiModalMixin ): ... diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index 62f6d684e8..ff8fc98dbc 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -35,7 +35,7 @@ from transformers.conversion_mapping import ( get_model_conversion_mapping, ) -from vllm.compilation.decorators import support_torch_compile +from vllm.compilation.decorators import DynamicArgDims, support_torch_compile from vllm.config.utils import getattr_iter from vllm.distributed import get_pp_group, get_tp_group from vllm.distributed.utils import get_pp_indices @@ -76,6 +76,7 @@ from vllm.model_executor.models.transformers.utils import ( from vllm.model_executor.models.utils import ( AutoWeightsLoader, PPMissingLayer, + ShardId, WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix, @@ -110,10 +111,13 @@ class Base( SupportsEagle, SupportsEagle3, ): - embedding_modules = ["embed_tokens"] # TODO transformers will have a util to get it + hf_to_vllm_mapper: WeightsMapper + + # TODO transformers will have a util to get these + embedding_modules = {"embed_tokens": "input_embeddings"} def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): - super().__init__() + nn.Module.__init__(self) logger.info("Using Transformers modeling backend.") self.vllm_config = vllm_config @@ -248,7 +252,7 @@ class Base( def _decorate_cls_for_torch_compile( self, cls: type["PreTrainedModel"], - dynamic_arg_dims: dict[str, int] | None, + dynamic_arg_dims: DynamicArgDims | None, enable_if: Callable[["VllmConfig"], bool], is_encoder: bool, ): @@ -284,7 +288,7 @@ class Base( self._decorate_cls_for_torch_compile( cls=self._pre_trained_model_classes.decoder, # Applied to a PreTrainedModel so the batch dimension will exist - dynamic_arg_dims=dict[str, int]( + dynamic_arg_dims=DynamicArgDims( input_ids=1, # shape: [1, seq_len] inputs_embeds=1, # shape: [1, seq_len, hidden_size] position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope @@ -303,9 +307,8 @@ class Base( - Checkpoints saved with no base model prefix - Any quantization config specific mappings """ - self.hf_to_vllm_mapper = WeightsMapper() - orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming - orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex + orig_to_new_renaming: list[WeightRenaming] = [] + orig_to_new_regex: dict[re.Pattern, str | None] = {} for mapping in get_model_conversion_mapping(self.model): # Handle weights which have been renamed in Transformers @@ -341,6 +344,11 @@ class Base( nested_lm_head_pattern = re.compile(r"^model\.(.+\.)*(lm_head.+)") orig_to_new_regex[nested_lm_head_pattern] = r"\2" + self.hf_to_vllm_mapper = WeightsMapper( + orig_to_new_renaming=orig_to_new_renaming, + orig_to_new_regex=orig_to_new_regex, + ) + # Apply mapping to quantization config if needed self._maybe_apply_model_mapping() @@ -371,8 +379,8 @@ class Base( "modeling backend will infer the split from the layers of %s in order " "of declaration and keep parameter-free modules on every rank. This " "may fail if the model's structure is non-standard. %s", - type(self.model), - type(module), + type(self.model).__name__, + type(module).__name__, tip, ) @@ -470,7 +478,7 @@ class Base( "backend will shard the model the best it can during graph fusion and " "replicate the rest. This may be suboptimal or fail if the model does " "not fuse cleanly. %s", - type(self.model), + type(self.model).__name__, tip, ) @@ -481,6 +489,8 @@ class Base( vocab_embeddings = self._vocab_embeddings() + orig_to_new_stacked: dict[str, tuple[str, ShardId]] = {} + def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module): """Register a fused layer's mappings just before it is built.""" self.fusers.setdefault(prefix, []).append(fuser) @@ -498,8 +508,7 @@ class Base( ) self.attention_fusers[index] = (prefix, fuser) - orig_to_new_stacked = fuser.orig_to_new_stacked(prefix) - self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked) + orig_to_new_stacked.update(fuser.orig_to_new_stacked(prefix)) packed_modules_mapping = fuser.packed_modules_mapping self.packed_modules_mapping.update(packed_modules_mapping) @@ -564,6 +573,8 @@ class Base( _recursive_replace(self.model, prefix="model") + self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked) + def _create_attention_instances(self): """Create `Attention` instances to inform KV cache allocation.""" text_config = self.text_config @@ -589,8 +600,11 @@ class Base( for i in range(start, end): if i not in self.attention_fusers: - in_range = layer_types and i < len(layer_types) - layer = f"{i} ({layer_types[i]})" if in_range else str(i) + layer = ( + f"{i} ({layer_types[i]})" + if layer_types and i < len(layer_types) + else str(i) + ) raise ValueError( f"Layer {layer} does not dispatch through the Transformers " "attention interface and vLLM has no other way to handle it." diff --git a/vllm/model_executor/models/transformers/causal.py b/vllm/model_executor/models/transformers/causal.py index c5231f9e97..e7dba82b0f 100644 --- a/vllm/model_executor/models/transformers/causal.py +++ b/vllm/model_executor/models/transformers/causal.py @@ -27,13 +27,15 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix +from .base import Base + if TYPE_CHECKING: import torch from vllm.config import VllmConfig -class CausalMixin(VllmModelForTextGeneration): +class CausalMixin(VllmModelForTextGeneration, Base): def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): # Skip VllmModelForTextGeneration.__init__ and call the next class in MRO super(VllmModelForTextGeneration, self).__init__( diff --git a/vllm/model_executor/models/transformers/fusers/attention.py b/vllm/model_executor/models/transformers/fusers/attention.py index fc16527f15..d8d324536d 100644 --- a/vllm/model_executor/models/transformers/fusers/attention.py +++ b/vllm/model_executor/models/transformers/fusers/attention.py @@ -5,6 +5,7 @@ import ast import inspect import textwrap +from collections.abc import Callable from dataclasses import dataclass from functools import cache from typing import TYPE_CHECKING, ClassVar @@ -36,16 +37,16 @@ def _is_interface_lookup(node: ast.expr | None) -> bool: @cache -def interface_call(cls: type[nn.Module]) -> ast.Call | None: - """The attention interface call in `cls.forward`, if it makes exactly one.""" +def interface_call(forward: Callable) -> ast.Call | None: + """The attention interface call in `forward`, if it makes exactly one.""" try: - source = inspect.getsource(inspect.unwrap(cls.forward)) + source = inspect.getsource(inspect.unwrap(forward)) tree = ast.parse(textwrap.dedent(source)) except (AttributeError, OSError, SyntaxError, TypeError): return None # Find the names of the local variables that read from the interface lookup - names = set() + names: set[str] = set() for node in ast.walk(tree): if not isinstance(node, (ast.Assign, ast.AnnAssign)): continue @@ -97,7 +98,7 @@ class AttentionFuser(BaseFuser): def match( cls, graph: fx.Graph | None, module: nn.Module ) -> "AttentionFuser | None": - if (call := interface_call(type(module))) is None: + if (call := interface_call(type(module).forward)) is None: return None scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"] scale_expr = scaling[0] if len(scaling) == 1 else None @@ -120,6 +121,7 @@ class AttentionFuser(BaseFuser): self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" ) -> nn.Module: if (sinks := self.sinks(module)) is not None: + assert isinstance(self.s_aux_expr, ast.Attribute) size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size device = vllm_config.device_config.device data = torch.empty(size, dtype=sinks.dtype, device=device) diff --git a/vllm/model_executor/models/transformers/fusers/base.py b/vllm/model_executor/models/transformers/fusers/base.py index ad24f2542a..937b70c5ed 100644 --- a/vllm/model_executor/models/transformers/fusers/base.py +++ b/vllm/model_executor/models/transformers/fusers/base.py @@ -130,11 +130,14 @@ class StackedFuser(RewriteFuser): """A fuser that merges sibling projections into one stacked linear and rewrites the forward to call it.""" - merged_name: ClassVar[str] - """Attribute name of the merged module created by `update_attrs`.""" merged_cls_name: ClassVar[str] """Name of the vLLM class the merged projection becomes (for logging).""" + @property + @abstractmethod + def merged_name(self) -> str: + """Attribute name of the merged module created by `update_attrs`.""" + def info(self, name: str) -> str: sources = " + ".join(shard for shard, _ in self.shards) return ( diff --git a/vllm/model_executor/models/transformers/fusers/mla.py b/vllm/model_executor/models/transformers/fusers/mla.py index fa4b34750c..96a4a0b87b 100644 --- a/vllm/model_executor/models/transformers/fusers/mla.py +++ b/vllm/model_executor/models/transformers/fusers/mla.py @@ -24,6 +24,7 @@ from vllm.model_executor.models.transformers.fx_utils import ( upstream_linear, ) from vllm.model_executor.models.transformers.utils import ( + Style, log_replacement, replace_linear_class, ) @@ -128,30 +129,26 @@ class MLAFuser(StackedFuser): merged_name: ClassVar[str] = "fused_qkv_a_proj" merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear" - @property - def has_q_lora(self) -> bool: - return self.q_a_proj_name is not None - def info(self, name: str) -> str: info_str = ( f"Fused: {name} ({self.source_cls}) -> MLAAttention (attention interface)" ) - if self.has_q_lora: + if self.q_a_proj_name is not None: info_str += "; " + super().info(name).removeprefix("Fused: ") return info_str @property def shards(self) -> list[tuple[str, ShardId]]: """`q_a_proj` and `kv_a_proj_with_mqa` stack into one down-projection.""" - if self.has_q_lora: - return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)] - return [] + if self.q_a_proj_name is None: + return [] + return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)] @property def packed_modules_mapping(self) -> dict[str, list[str]]: - if self.has_q_lora: - return super().packed_modules_mapping - return {} + if self.q_a_proj_name is None: + return {} + return super().packed_modules_mapping @classmethod def match(cls, graph: fx.Graph, module: nn.Module) -> "MLAFuser | None": @@ -234,10 +231,10 @@ class MLAFuser(StackedFuser): Bypass the KV expansion method so the compressed latent reaches the `vllm_mla` attention interface unexpanded.""" funcdef, fn = recover_forward(type(module)) - if self.has_q_lora: + if (q_a_proj_name := self.q_a_proj_name) is not None: # q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`. # The fused call is inserted at the top-level statement preceding both. - names = [self.q_a_proj_name, self.kv_a_proj_name] + names = [q_a_proj_name, self.kv_a_proj_name] calls = self._unguarded_calls(funcdef, names) if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]): raise ValueError("down-projections read different inputs") @@ -258,7 +255,7 @@ class MLAFuser(StackedFuser): def update_attrs(self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"): quant_config = vllm_config.quant_config - def replace_linear_by_name(name: str, style: str): + def replace_linear_by_name(name: str, style: Style): linear = module.get_submodule(name) replacement = replace_linear_class( linear, style, quant_config, prefix=maybe_prefix(prefix, name) @@ -266,8 +263,10 @@ class MLAFuser(StackedFuser): setattr(module, name, replacement) log_replacement(maybe_prefix(prefix, name), linear, replacement) - if self.has_q_lora: - q_a = module.get_submodule(self.q_a_proj_name) + if (q_a_proj_name := self.q_a_proj_name) is not None: + # `match` sets the q-LoRA names together, or none of them + assert self.q_b_proj_name is not None + q_a = module.get_submodule(q_a_proj_name) kv_a = module.get_submodule(self.kv_a_proj_name) merged = MergedColumnParallelLinear( input_size=q_a.in_features, @@ -280,7 +279,7 @@ class MLAFuser(StackedFuser): ) logger.debug( "%s: %s, %s: %s -> %s: %s", - self.q_a_proj_name, + q_a_proj_name, q_a, self.kv_a_proj_name, kv_a, @@ -289,10 +288,11 @@ class MLAFuser(StackedFuser): ) setattr(module, self.merged_name, merged) # The rewritten forward calls the merged projection instead. - delattr(module, self.q_a_proj_name) + delattr(module, q_a_proj_name) delattr(module, self.kv_a_proj_name) replace_linear_by_name(self.q_b_proj_name, "colwise") else: + assert self.q_proj_name is not None replace_linear_by_name(self.kv_a_proj_name, "replicate") replace_linear_by_name(self.q_proj_name, "colwise") diff --git a/vllm/model_executor/models/transformers/fusers/rms_norm.py b/vllm/model_executor/models/transformers/fusers/rms_norm.py index dd2c5339ce..f7972f03a6 100644 --- a/vllm/model_executor/models/transformers/fusers/rms_norm.py +++ b/vllm/model_executor/models/transformers/fusers/rms_norm.py @@ -216,7 +216,7 @@ class RMSNormFuser(BaseFuser): finally: for name, value in candidates.items(): setattr(module, name, value) - if (name := markers.get(marked)) is not None: + if marked is not None and (name := markers.get(marked)) is not None: return name, None logger.debug_once( "%s does not hold its eps (%s) in an attribute. Every instance in this " @@ -251,18 +251,18 @@ class RMSNormFuser(BaseFuser): ) -> nn.Module: """Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp.""" weight = getattr(module, "weight", None) - has_weight = weight is not None - hidden_size = weight.size(0) if has_weight else 0 + hidden_size = weight.size(0) if weight is not None else 0 eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps if not isinstance(eps, (int, float)): # If eps was not detected, match torch behaviour. - dtype = weight.dtype if has_weight else vllm_config.model_config.dtype + config_dtype = vllm_config.model_config.dtype + dtype = weight.dtype if weight is not None else config_dtype eps = torch.finfo(dtype).eps if self.zero_centered: return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps) return TPAwareRMSNorm( hidden_size=hidden_size, eps=eps, - has_weight=has_weight, - dtype=weight.dtype if has_weight else None, + has_weight=weight is not None, + dtype=weight.dtype if weight is not None else None, ) diff --git a/vllm/model_executor/models/transformers/fx_utils.py b/vllm/model_executor/models/transformers/fx_utils.py index 3cdc4e1201..cb630866a3 100644 --- a/vllm/model_executor/models/transformers/fx_utils.py +++ b/vllm/model_executor/models/transformers/fx_utils.py @@ -15,17 +15,19 @@ import operator import textwrap from collections.abc import Callable from itertools import chain +from typing import Any from unittest import mock import torch from torch import fx, nn from torch.nn import functional as F +from typing_extensions import TypeIs from vllm.logger import init_logger logger = init_logger(__name__) -_UNKNOWN = object() +_UNKNOWN: Any = object() """Sentinel meta value for proxies whose concrete value could not be inferred. Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`).""" @@ -34,7 +36,7 @@ _MODULE_CALL = nn.Module.__call__ `call_module` nodes; meta execution must call modules for real.""" -def is_leaf_call(node: object) -> bool: +def is_leaf_call(node: object) -> TypeIs[fx.Node]: """Is node a call recorded by `_as_leaf_call` (e.g. an attention interface).""" return isinstance(node, fx.Node) and node.meta.get("leaf_call", False) @@ -60,7 +62,7 @@ class _MetaProxy(fx.Proxy): executing each op on the meta values, so PyTorch's meta kernels are the single source of shape inference — no per-op rules.""" - meta: object = _UNKNOWN + meta: Any = _UNKNOWN def __len__(self) -> int: if self.meta is not _UNKNOWN: @@ -149,6 +151,7 @@ class _AllLeafTracer(fx.Tracer): if unknown: return _UNKNOWN if kind == "call_function": + assert callable(target) return target(*meta_args, **meta_kwargs) if kind == "call_method": receiver, *rest = meta_args @@ -576,7 +579,7 @@ def _aliasing_reads(node: ast.expr) -> set[str]: would taint nearly every later name. """ names: set[str] = set() - stack = [node] + stack: list[ast.AST] = [node] while stack: current = stack.pop() if isinstance(current, ast.Attribute) and current.attr in _METADATA_ATTRS: @@ -646,7 +649,7 @@ def aliasing_names( """ names = set(seed) assigns = [ - node + (node, node.value) for node in ast.walk(funcdef) if isinstance(node, (ast.Assign, ast.AnnAssign)) and node.value is not None @@ -655,10 +658,10 @@ def aliasing_names( changed = True while changed: changed = False - for node in assigns: - if not _aliasing_reads(node.value) & names: + for node, value in assigns: + if not _aliasing_reads(value) & names: continue - targets = getattr(node, "targets", None) or [node.target] + targets = node.targets if isinstance(node, ast.Assign) else [node.target] stores = { child.id for target in targets @@ -770,7 +773,7 @@ def peel(node: object) -> object: return node -def is_fn(node: object, target: Callable) -> bool: +def is_fn(node: object, target: object) -> TypeIs[fx.Node]: """Is node `()`.""" return ( isinstance(node, fx.Node) @@ -779,14 +782,14 @@ def is_fn(node: object, target: Callable) -> bool: ) -def is_method(node: object, name: str) -> bool: +def is_method(node: object, name: str) -> TypeIs[fx.Node]: """Is node `.()`.""" return ( isinstance(node, fx.Node) and node.op == "call_method" and node.target == name ) -def is_op(node: object, name: str) -> bool: +def is_op(node: object, name: str) -> TypeIs[fx.Node]: """Is node `.()` for torch, F, operator, or Tensor.""" return any( is_fn(node, getattr(module, name, None)) for module in (torch, F, operator) diff --git a/vllm/model_executor/models/transformers/legacy.py b/vllm/model_executor/models/transformers/legacy.py index 9c8a5c45b6..7a256f29d5 100644 --- a/vllm/model_executor/models/transformers/legacy.py +++ b/vllm/model_executor/models/transformers/legacy.py @@ -20,35 +20,38 @@ from typing import TYPE_CHECKING import torch +from vllm.model_executor.models.utils import WeightsMapper from vllm.sequence import IntermediateTensors +from .base import Base + if TYPE_CHECKING: from vllm.config import VllmConfig -class LegacyMixin: +class LegacyMixin(Base): def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): super().__init__(vllm_config=vllm_config, prefix=prefix) - # Drop unsupported/unwanted output embeddings layers. - self.hf_to_vllm_mapper.orig_to_new_prefix.update( - { + self.hf_to_vllm_mapper |= WeightsMapper( + # Drop unsupported/unwanted output embeddings layers. + orig_to_new_prefix={ "model.lm_head.": None, "model.predictions.": None, "model.qa_outputs.": None, "model.embeddings_project.": None, "model.discriminator_predictions.": None, - } + }, + orig_to_new_substr={ + # Some encoder models have the position_ids buffer in the checkpoint. + # vLLM always passes position_ids as an argument, so drop the buffer. + "position_ids": None, + # Some encoder models have the bias of the final classifier layer in + # the checkpoint. vLLM does not use this bias, so drop it. + "score.bias": None, + }, ) - # Some encoder models have the position_ids buffer in the checkpoint. vLLM will - # always pass position_ids as an argument, so we drop the buffer if it exists. - self.hf_to_vllm_mapper.orig_to_new_substr["position_ids"] = None - - # Some encoder models have the bias of the final classifier layer in the - # checkpoint. vLLM does not use this bias, so we drop it if it exists. - self.hf_to_vllm_mapper.orig_to_new_substr["score.bias"] = None - # roberta-like models an extra padding in positions. # FIXME(Isotr0py): This is quite hacky for roberta edge case, # we should find a better way to handle this. @@ -61,6 +64,7 @@ class LegacyMixin: positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, + **kwargs, ) -> torch.Tensor | IntermediateTensors: if self.is_roberta: # RoBERTa positions start at padding_idx + 1. @@ -72,4 +76,5 @@ class LegacyMixin: positions=positions, intermediate_tensors=intermediate_tensors, inputs_embeds=inputs_embeds, + **kwargs, ) diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index 480bc7f66e..9f63e73336 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -41,6 +41,7 @@ from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op +from .base import Base from .utils import log_replacement, maybe_per_layer if TYPE_CHECKING: @@ -66,14 +67,15 @@ class TransformersMoERunner(MoERunner): self._moe_state = moe_state self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel - def forward( + def forward( # type: ignore[override] self, hidden_states: torch.Tensor, topk_ids: torch.Tensor, topk_weights: torch.Tensor, **kwargs: Any, ) -> torch.Tensor: - """In Transformers `experts.forward` will have this signature. + """In Transformers `experts.forward` will have this signature, which is why + it deliberately does not match `MoERunner.forward`. We discard any extra kwargs because we cannot use them here.""" # Note: we need to forward through a custom op so the topk_ids @@ -126,10 +128,18 @@ direct_register_custom_op( class TransformersRoutedExperts(RoutedExperts): def get_expert_mapping( - self, include_fused: bool = False + self, + ckpt_gate_proj_name: str | None = None, + ckpt_down_proj_name: str | None = None, + ckpt_up_proj_name: str | None = None, + include_fused: bool = False, ) -> list[tuple[str, str, int, str]]: - common_names = ("gate_proj", "down_proj", "up_proj") - common_map = super().get_expert_mapping(*common_names, include_fused) + common_map = super().get_expert_mapping( + ckpt_gate_proj_name or "gate_proj", + ckpt_down_proj_name or "down_proj", + ckpt_up_proj_name or "up_proj", + include_fused, + ) mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused) if not include_fused: return common_map + mixtral_map @@ -138,7 +148,13 @@ class TransformersRoutedExperts(RoutedExperts): return common_fused + mixtral_fused + common_unfused + mixtral_unfused -class MoEMixin(MixtureOfExperts): +class MoEMixin(MixtureOfExperts, Base): + mlp_layers: list[nn.Module] + """MoE blocks whose experts were replaced, for the `MixtureOfExperts` methods.""" + + # Redeclared because `recursive_replace` assigns it below the first read + num_local_physical_experts: int + def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): self.check_version("5.0.0", "MoE models support") # Skip MixtureOfExperts.__init__ and call the next class in MRO @@ -197,7 +213,8 @@ class MoEMixin(MixtureOfExperts): # Dtype the router computes in, if it is not the activation dtype. config_router_dtype = getattr(text_config, "moe_router_dtype", None) - config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype) + if config_router_dtype is not None: + config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype) # Grouped topk routing kwargs num_expert_group = getattr(text_config, "n_group", None) @@ -222,7 +239,7 @@ class MoEMixin(MixtureOfExperts): # MixtureOfExperts mixin settings ep_size = get_ep_group().world_size - self.mlp_layers = [] # Used for MixtureOfExperts methods + self.mlp_layers = [] self.moe_layers = [] self.num_expert_groups = 1 if num_expert_group is None else num_expert_group self.num_logical_experts = num_experts @@ -321,9 +338,10 @@ class MoEMixin(MixtureOfExperts): if shared_experts is not None: hf_shared = shared_experts.shared_experts glu_fuser = get_fuser(hf_shared, GLUFuser) - down_name = glu_fuser and glu_fuser.down_name - if down_name is not None: - shared_down_projs.append((hf_shared, down_name)) + if glu_fuser is not None: + down_name = glu_fuser.down_name + if down_name is not None: + shared_down_projs.append((hf_shared, down_name)) # Prefer config, otherwise read it from fuser. router_dtype = config_router_dtype or fuser.router_dtype gate = fuser.gate(moe_block, prefix, router_dtype) @@ -374,7 +392,9 @@ class MoEMixin(MixtureOfExperts): # Handle all gather in expert parallel if topk_ids.size(0) != hidden_states.size(0): dp_metadata = get_forward_context().dp_metadata + assert dp_metadata is not None sizes = dp_metadata.get_chunk_sizes_across_dp_rank() + assert sizes is not None is_sp = moe_state.is_sequence_parallel group = get_ep_group() if is_sp else get_dp_group() assert sizes[group.rank_in_group] == topk_ids.shape[0] diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index 855b267c02..164a4dbd83 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -17,17 +17,24 @@ """Transformers modeling backend mixin for multi-modal models.""" from collections import defaultdict -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from contextlib import ExitStack, contextmanager from typing import TYPE_CHECKING, Any import torch import transformers from packaging.version import Version +from transformers.utils.generic import ModelOutput from vllm.compilation.decorators import should_torch_compile_mm_encoder +from vllm.config.multimodal import AudioDummyOptions from vllm.config.utils import getattr_iter -from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input +from vllm.inputs import ( + MultiModalDataBuiltins, + MultiModalDataDict, + MultiModalInput, + mm_input, +) from vllm.logger import init_logger from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, @@ -62,6 +69,8 @@ from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import async_tensor_h2d +from .base import Base + if TYPE_CHECKING: from transformers import BatchFeature, PreTrainedModel @@ -148,7 +157,7 @@ class MultiModalProcessingInfo(BaseProcessingInfo): def get_max_image_tokens(self) -> int: width, height = self.get_image_size_with_most_features() processor = self.get_hf_processor() - multimodal_config = self.ctx.model_config.multimodal_config + multimodal_config = self.ctx.model_config.get_multimodal_config() mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {} mm_tokens = processor._get_num_multimodal_tokens( image_sizes=([height, width],), **mm_processor_kwargs @@ -188,7 +197,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn mm_counts: Mapping[str, int], mm_options: Mapping[str, "BaseDummyOptions"], ) -> MultiModalDataDict: - data: MultiModalDataDict = {} + data = MultiModalDataBuiltins() if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)): sampling_rate = self.info._get_audio_sampling_rate() sub = self.info._get_audio_processor() @@ -196,10 +205,14 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn if chunk_length is None: chunk_length = 30 audio_len = int(chunk_length * sampling_rate) + audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance( + audio_overrides, AudioDummyOptions + ) data["audio"] = self._get_dummy_audios( length=audio_len, num_audios=num_audios, - overrides=mm_options.get("audio"), + overrides=audio_overrides, ) if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)): target_width, target_height = self.info.get_image_size_with_most_features() @@ -359,7 +372,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo] itself is to process it by itself. """ audios = mm_data.get("audio") - if not audios or len({len(audio) for audio in audios}) == 1: + if not isinstance(audios, Iterable) or not audios: + return + if len({len(audio) for audio in audios}) == 1: return alone = [ @@ -444,7 +459,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase): token_id = getattr(processor, names[0], getattr_iter(config, names)) if token_id is None: token = getattr(processor, f"{modality}_token", None) - token_id = info.get_tokenizer().get_vocab().get(token) + if token is not None: + token_id = info.get_tokenizer().get_vocab().get(token) if token_id is None: raise ValueError( f"Cannot find {modality}_token_id on processor or model config" @@ -722,13 +738,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): for modality, items in out_mm_kwargs.items(): # Popped so they are neither cached nor sent to the model; the updates # they produce are cached alongside the item instead - replacements = [ - PromptUpdateDetails.select_token_id( - (ids := item.pop(f"{modality}_replacement_ids").data).tolist(), - _get_embed_token_id(ids), + replacements = [] + for item in items: + ids = item.pop(f"{modality}_replacement_ids").data + assert isinstance(ids, torch.Tensor) + replacements.append( + PromptUpdateDetails.select_token_id( + ids.tolist(), _get_embed_token_id(ids) + ) ) - for item in items - ] token = getattr(hf_processor, f"{modality}_token") updates.append( PromptReplacement( @@ -777,7 +795,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): ) -> list[int] | None: """Ask the HF processor how many rows of image data each image produces.""" images = mm_data.get("images") - if not images: + if not isinstance(images, Iterable) or not images: return None try: sizes = [(image.height, image.width) for image in images] @@ -893,7 +911,7 @@ MultiModalProcessor = ( ) -class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): +class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base): def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): # Skip SupportsMRoPE.__init__ and call the next class in MRO super(SupportsMRoPE, self).__init__(vllm_config=vllm_config, prefix=prefix) @@ -925,7 +943,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): if model_config.skip_tokenizer_init: # Determining the supported modalities needs the HF processor, which in # turn needs a tokenizer - mm_config = model_config.multimodal_config + mm_config = model_config.get_multimodal_config() if mm_config.mm_encoder_only or any( mm_config.get_limit_per_prompt(modality) == 0 for modality in encoder_classes @@ -1029,7 +1047,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): # Exclude MultiModalMixin itself bases = [b for b in bases if b is not MultiModalMixin] - class LanguageModel(*bases): + class LanguageModel(*bases): # type: ignore[misc] def __init__(self, multimodal_model): # Don't call super().__init__() to avoid re-initialization self.__dict__.update(multimodal_model.__dict__) @@ -1203,7 +1221,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): # from `self.get_image_features` if isinstance(features, tuple): return features[0] - if isinstance(features, dict): + if isinstance(features, ModelOutput): return features.pooler_output return features diff --git a/vllm/model_executor/models/transformers/pooling.py b/vllm/model_executor/models/transformers/pooling.py index 47a9c23d2c..79594387b7 100644 --- a/vllm/model_executor/models/transformers/pooling.py +++ b/vllm/model_executor/models/transformers/pooling.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING import torch +import torch.nn as nn from transformers import AutoModelForSequenceClassification from vllm.config.utils import getattr_iter @@ -26,11 +27,25 @@ from vllm.model_executor.layers.pooler import DispatchPooler from vllm.model_executor.models.interfaces import SupportsCrossEncoding from vllm.model_executor.models.interfaces_base import VllmModelForPooling +from .base import Base + if TYPE_CHECKING: from vllm.config import VllmConfig -class EmbeddingMixin(VllmModelForPooling): +class ClassifierWithReshape(nn.Module): + """Token extraction has already been applied in `pooler.pooling`. + + Add dim to match expected input shape of `classifier.forward`. + """ + + def forward(self, *args, **kwargs): + if len(args) > 0: + args = (args[0].unsqueeze(1), *args[1:]) + return super().forward(*args, **kwargs) + + +class EmbeddingMixin(VllmModelForPooling, Base): default_seq_pooling_type = "CLS" def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): @@ -45,7 +60,7 @@ class EmbeddingMixin(VllmModelForPooling): self.pooler = DispatchPooler.for_embedding(pooler_config) -class SequenceClassificationMixin(SupportsCrossEncoding, VllmModelForPooling): +class SequenceClassificationMixin(SupportsCrossEncoding, VllmModelForPooling, Base): default_seq_pooling_type = "CLS" def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): @@ -83,17 +98,13 @@ class SequenceClassificationMixin(SupportsCrossEncoding, VllmModelForPooling): ) self.init_parameters(self.classifier, dtype=self.model_config.head_dtype) - class ClassifierWithReshape(self.classifier.__class__): - """Token extraction has already been applied in `pooler.pooling`. - Add dim to match expected input shape of `classifier.forward`. - """ - - def forward(self, *args, **kwargs): - if len(args) > 0: - args = (args[0].unsqueeze(1), *args[1:]) - return super().forward(*args, **kwargs) - - self.classifier.__class__ = ClassifierWithReshape + # Order `ClassifierWithReshape` ahead of the classifier's own class so that + # its `super().forward(...)` reaches the original implementation. + self.classifier.__class__ = type( + "ClassifierWithReshape", + (ClassifierWithReshape, type(self.classifier)), + {}, + ) self.pooler = DispatchPooler.for_seq_cls( pooler_config, diff --git a/vllm/model_executor/models/transformers/utils.py b/vllm/model_executor/models/transformers/utils.py index ddb2f4fb71..0443f390c4 100644 --- a/vllm/model_executor/models/transformers/utils.py +++ b/vllm/model_executor/models/transformers/utils.py @@ -18,11 +18,10 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager -from functools import lru_cache from itertools import chain from operator import attrgetter from pathlib import Path -from typing import TYPE_CHECKING, Literal, TypeVar +from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar import torch from torch import nn @@ -70,7 +69,7 @@ def init_on_device_without_buffers(device: torch.device): module._parameters[name].to(device), **kwargs ) - tensor_constructors_to_patch = {} + tensor_constructors_to_patch: dict[str, Callable] = {} def patch_tensor_constructor(fn): def wrapper(*args, **kwargs): @@ -147,8 +146,8 @@ def replace_linear_class( ) -TorchConv = nn.Conv2d | nn.Conv3d -VllmConv = Conv2dLayer | Conv3dLayer +TorchConv: TypeAlias = nn.Conv2d | nn.Conv3d +VllmConv: TypeAlias = Conv2dLayer | Conv3dLayer def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv: @@ -212,7 +211,9 @@ class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbeddin `super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding.""" -@lru_cache +_rebased_embedding_classes: dict[type[nn.Embedding], type[VocabParallelEmbedding]] = {} + + def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]: """Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`. @@ -224,7 +225,10 @@ def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbe The new class, to assign to `__class__` of an instance of `cls`. """ - return type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {}) + if cls not in _rebased_embedding_classes: + rebased_cls = type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {}) + _rebased_embedding_classes[cls] = rebased_cls + return _rebased_embedding_classes[cls] def replace_embedding_class( @@ -278,10 +282,9 @@ def recursive_replace_linear( qual_name = maybe_prefix(prefix, child_name) # Replace modules as needed if isinstance(child_module, nn.Linear): - style = "replicate" new_module = replace_linear_class( child_module, - style, + "replicate", quant_config, prefix=qual_name, ) From fef55fd26dc633fcc87978f12f1a97e08e351c02 Mon Sep 17 00:00:00 2001 From: Matt <156021403+mawong-amd@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:21:22 -0700 Subject: [PATCH 043/143] [ROCm][CI] Adapt MoE tests to the triton_kernels 3.8 API (#57385) Signed-off-by: Matthew Wong Co-authored-by: Claude Opus 5 (1M context) --- .../moe/test_gpt_oss_triton_kernels.py | 38 ++++++++++++------- .../moe/test_modular_oai_triton_moe.py | 26 ++++++++----- tests/kernels/moe/utils.py | 24 ++++++++++++ .../layers/fused_moe/oracle/mxfp4.py | 16 +++----- .../layers/quantization/utils/mxfp4_utils.py | 9 +++++ 5 files changed, 79 insertions(+), 34 deletions(-) diff --git a/tests/kernels/moe/test_gpt_oss_triton_kernels.py b/tests/kernels/moe/test_gpt_oss_triton_kernels.py index 86028a7feb..c1d83d8349 100644 --- a/tests/kernels/moe/test_gpt_oss_triton_kernels.py +++ b/tests/kernels/moe/test_gpt_oss_triton_kernels.py @@ -7,7 +7,7 @@ import torch import torch.nn.functional as F from vllm.platforms import current_platform -from vllm.utils.import_utils import has_triton_kernels +from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels if not has_triton_kernels(): pytest.skip( @@ -15,9 +15,15 @@ if not has_triton_kernels(): allow_module_level=True, ) -import triton_kernels.matmul_ogs_details.opt_flags as opt_flags import triton_kernels.swiglu -from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig + +if get_triton_kernels_version() == "3.8": + # 3.8: matmul_ogs -> matmul, matmul_ogs_details -> matmul_details + import triton_kernels.matmul_details.opt_flags as opt_flags + from triton_kernels.matmul import FlexCtx, PrecisionConfig +else: + import triton_kernels.matmul_ogs_details.opt_flags as opt_flags + from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig from triton_kernels.numerics import InFlexData from triton_kernels.numerics_details.mxfp import downcast_to_mxfp, upcast_from_mxfp from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor @@ -28,9 +34,10 @@ from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_co from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( triton_kernel_moe_forward, ) +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs from vllm.utils.math_utils import round_up -from .utils import shuffle_weight +from .utils import mxfp4_w_layouts, shuffle_weight def deshuffle(w: torch.Tensor): @@ -133,12 +140,12 @@ def init_compute_data(M, K, N, E, a_dtype: str, w_dtype: str, num_warps: int): x_tri = F.pad(x_tri, (0, x_pad, 0, 0), mode="constant", value=0) - w_layout, w_layout_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=1) - w_scale_layout, w_scale_layout_opts = ( - layout.make_default_matmul_mxfp4_w_scale_layout( - mx_axis=1, num_warps=num_warps - ) - ) + ( + w_layout, + w_layout_opts, + w_scale_layout, + w_scale_layout_opts, + ) = mxfp4_w_layouts(mx_axis=1, num_warps=num_warps) w1_tri, w1_scale_tri = downcast_to_mxfp(w1_tri, torch.uint8, axis=1) w1 = upcast_from_mxfp(w1_tri, w1_scale_tri, torch.bfloat16, axis=1) @@ -165,10 +172,10 @@ def init_compute_data(M, K, N, E, a_dtype: str, w_dtype: str, num_warps: int): ) pc1 = PrecisionConfig( - weight_scale=w1_scale_tri, flex_ctx=FlexCtx(rhs_data=InFlexData()) + **mx_scale_kwargs(w1_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData()) ) pc2 = PrecisionConfig( - weight_scale=w2_scale_tri, flex_ctx=FlexCtx(rhs_data=InFlexData()) + **mx_scale_kwargs(w2_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData()) ) # tucuate so the rest can run properly @@ -281,8 +288,6 @@ class Case: @pytest.mark.parametrize("num_token", [2]) @pytest.mark.parametrize("tp", [1, 2, 4, 8]) def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init): - from triton_kernels.tensor_details import layout - if not hasattr(layout, "make_default_matmul_mxfp4_w_layout"): pytest.skip("make_default_matmul_mxfp4_w_layout not available") @@ -387,6 +392,11 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk): if gptoss_moe.triton_kernels_version == "3.5.1": pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+") + if gptoss_moe.triton_kernels_version == "3.8": + pytest.skip( + "3.8 rebuilds routing from raw ids, so routing_data_from_sparse_topk " + "just forwards to make_routing_data and the comparison is vacuous" + ) from triton_kernels.topk import topk as topk_fn diff --git a/tests/kernels/moe/test_modular_oai_triton_moe.py b/tests/kernels/moe/test_modular_oai_triton_moe.py index 47be82dd2a..9665079e37 100644 --- a/tests/kernels/moe/test_modular_oai_triton_moe.py +++ b/tests/kernels/moe/test_modular_oai_triton_moe.py @@ -10,7 +10,7 @@ import torch.nn.functional as F from tests.utils import wait_for_gpu_memory_to_clear from vllm.model_executor.layers.fused_moe.activation import MoEActivation -from vllm.utils.import_utils import has_triton_kernels +from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels if not has_triton_kernels(): pytest.skip( @@ -18,11 +18,14 @@ if not has_triton_kernels(): allow_module_level=True, ) -from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig +if get_triton_kernels_version() == "3.8": + # 3.8: matmul_ogs -> matmul + from triton_kernels.matmul import FlexCtx, PrecisionConfig +else: + from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig from triton_kernels.numerics import InFlexData from triton_kernels.numerics_details.mxfp import downcast_to_mxfp, upcast_from_mxfp from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor -from triton_kernels.tensor_details import layout from triton_kernels.testing import assert_close from vllm.config import VllmConfig, set_current_vllm_config @@ -35,11 +38,12 @@ from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe imp UnfusedOAITritonExperts, ) from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs from vllm.platforms import current_platform from vllm.utils.math_utils import round_up from vllm.utils.torch_utils import set_random_seed -from .utils import make_dummy_moe_config, shuffle_weight +from .utils import make_dummy_moe_config, mxfp4_w_layouts, shuffle_weight MNK = [ (1, 512, 384), @@ -130,10 +134,12 @@ def make_weights(dtype, k, n, e): w2 = w2[..., :n, :k] num_warps = 8 - w_layout, w_layout_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=1) - w_scale_layout, w_scale_layout_opts = ( - layout.make_default_matmul_mxfp4_w_scale_layout(mx_axis=1, num_warps=num_warps) - ) + ( + w_layout, + w_layout_opts, + w_scale_layout, + w_scale_layout_opts, + ) = mxfp4_w_layouts(mx_axis=1, num_warps=num_warps) w1_tri = convert_layout(wrap_torch_tensor(w1_tri, FP4), w_layout, **w_layout_opts) w1_scale_tri = convert_layout( @@ -150,10 +156,10 @@ def make_weights(dtype, k, n, e): ) w1_precision_config = PrecisionConfig( - weight_scale=w1_scale_tri, flex_ctx=FlexCtx(rhs_data=InFlexData()) + **mx_scale_kwargs(w1_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData()) ) w2_precision_config = PrecisionConfig( - weight_scale=w2_scale_tri, flex_ctx=FlexCtx(rhs_data=InFlexData()) + **mx_scale_kwargs(w2_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData()) ) return ( diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index e762472950..d92e4d6129 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -705,3 +705,27 @@ def check_accuracy(a, b, atol, rtol, percent): f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} " f"(threshold: {1 - percent:.4f})" ) + + +def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8): + """Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs. + + triton_kernels 3.8 returns layout instances; earlier versions return a + (layout, opts) tuple. + """ + from triton_kernels.tensor_details import layout + + from vllm.utils.import_utils import get_triton_kernels_version + + if get_triton_kernels_version() == "3.8": + w = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis) + s = layout.make_default_matmul_mxfp4_w_scale_layout( + mx_axis=mx_axis, num_warps=num_warps + ) + return w, {}, s, {} + + w, w_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis) + s, s_opts = layout.make_default_matmul_mxfp4_w_scale_layout( + mx_axis=mx_axis, num_warps=num_warps + ) + return w, w_opts, s, s_opts diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index d1c151aca2..7fed4124f5 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -31,7 +31,12 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( swap_w13_to_w31, ) -from vllm.model_executor.layers.quantization.utils.mxfp4_utils import _swizzle_mxfp4 +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + _swizzle_mxfp4, +) +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + mx_scale_kwargs as _mx_scale_kwargs, +) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( OCP_MX_BLOCK_SIZE, ) @@ -70,15 +75,6 @@ if triton_kernels_version is not None: ) -def _mx_scale_kwargs(scale): - """PrecisionConfig weight-scale kwargs: 3.8 uses b_mx_scale/b_microblock_size, - 3.5.1/3.6 use weight_scale. - """ - if triton_kernels_version == "3.8": - return {"b_mx_scale": scale, "b_microblock_size": 32} - return {"weight_scale": scale} - - def _pack_deepgemm_mxfp4_scales( w13_weight: torch.Tensor, w2_weight: torch.Tensor, diff --git a/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py b/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py index a1d0501775..8cc626f386 100644 --- a/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py +++ b/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py @@ -36,6 +36,15 @@ def weight_mx_scale(precision_config): return precision_config.weight_scale +def mx_scale_kwargs(scale): + """PrecisionConfig weight-scale kwargs: 3.8 uses b_mx_scale/b_microblock_size, + 3.5.1/3.6 use weight_scale. + """ + if get_triton_kernels_version() == "3.8": + return {"b_mx_scale": scale, "b_microblock_size": 32} + return {"weight_scale": scale} + + def _swizzle_mxfp4(quant_tensor, scale, num_warps=8): """Weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel.""" assert has_triton_kernels() From e30bf70c5dfb90fd9b31a8f4692b49df9b223704 Mon Sep 17 00:00:00 2001 From: Matt <156021403+mawong-amd@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:22:39 -0700 Subject: [PATCH 044/143] [ROCm][CI] Fix Entrypoints Integration (Pooling) tests on TheRock image (#57380) Signed-off-by: Matthew Wong Co-authored-by: Claude Opus 5 (1M context) --- .../test_cross_encoder_online_vision.py | 58 ++++++------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py b/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py index 9f2d7da801..bf3c771fdd 100644 --- a/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py +++ b/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py @@ -27,31 +27,10 @@ ROCM_ATTN_BACKENDS = [ ATTN_BACKENDS = ROCM_ATTN_BACKENDS if current_platform.is_rocm() else ["auto"] -# Per-backend tolerance with explicit entries; "default" is the fallback -BACKEND_TOL: dict[str, float] = { - "default": 0.05, # 5% tolerance for other backends (e.g. FLASH_ATTN) - # Relaxed tolerances for ROCm attn - # See: https://github.com/vllm-project/vllm/issues/35569 - "ROCM_ATTN": 0.09, # gfx950:~8.45%, gfx942:~3.70% - "ROCM_AITER_FA": 0.045, # gfx950:~2.00%, gfx942:~0.80% - "TRITON_ATTN": 0.045, # gfx950:~3.00%, gfx942:~2.20% - "FLEX_ATTENTION": 0.045, # gfx950:~3.25%, gfx942:~1.10% -} - -# Some ROCm attention backends show small absolute drift on the low -# text-vs-text probability even though larger scores remain well inside the -# relative tolerance. The absolute drift is uniform across score magnitudes -# (~0.005-0.010), so it only exceeds the relative tolerance for the small -# ~0.10 text-vs-text value. Keep the relative tolerances tight and add only a -# small absolute floor for the affected backends. -# TRITON_ATTN: gfx942/ROCm 7.2 drifts ~0.008 abs on text-vs-text (~7.9% rel). -BACKEND_ABS_TOL: dict[str, float] = { - "default": 0.0, - "auto": 0.007, - "ROCM_AITER_FA": 0.005, - "TRITON_ATTN": 0.009, - "FLEX_ATTENTION": 0.00875, -} +# The absolute floor matters only for the low ~0.10 text-vs-text score, where a +# few thousandths of drift would otherwise exceed the relative bound. +REL_TOL = 0.05 +ABS_TOL = 0.007 # ROCm: disable prefix caching and eliminate batch variance to reduce # test flakiness. @@ -62,17 +41,9 @@ ROCM_EXTRA_ARGS = ( ) -def get_tol(backend: str) -> float: - return BACKEND_TOL.get(backend, BACKEND_TOL["default"]) - - -def get_abs_tol(backend: str) -> float: - return BACKEND_ABS_TOL.get(backend, BACKEND_ABS_TOL["default"]) - - def assert_score(actual: float, expected: float, backend: str, label: str): - tol = get_tol(backend) - abs_tol = get_abs_tol(backend) + tol = REL_TOL + abs_tol = ABS_TOL diff = abs(actual - expected) rel_diff = diff / abs(expected) if expected != 0 else diff print( @@ -104,9 +75,16 @@ documents = [ }, ] -TEXT_VS_TEXT = 0.10040374100208282 -TEXT_VS_IMAGE = 0.7423753142356873 -TEXT_VS_TEXT_PLUS_IMAGE = 0.5298863053321838 +# One bf16 ULP in attention compounds to ~0.1% here, so ROCm backends each +# settle differently; these are their midpoint across backends and archs. +if current_platform.is_rocm(): + TEXT_VS_TEXT = 0.105243 + TEXT_VS_IMAGE = 0.742472 + TEXT_VS_TEXT_PLUS_IMAGE = 0.544568 +else: + TEXT_VS_TEXT = 0.10040374100208282 + TEXT_VS_IMAGE = 0.7423753142356873 + TEXT_VS_TEXT_PLUS_IMAGE = 0.5298863053321838 @pytest.fixture(scope="module", params=ATTN_BACKENDS) @@ -507,6 +485,4 @@ async def test_rerank_api_instruction_field_matches_chat_template_kwargs( kwargs_scores = [ r.relevance_score for r in sorted(kwargs_rerank.results, key=lambda x: x.index) ] - assert field_scores == pytest.approx( - kwargs_scores, rel=get_tol(backend), abs=get_abs_tol(backend) - ) + assert field_scores == pytest.approx(kwargs_scores, rel=REL_TOL, abs=ABS_TOL) From b2f2bd71ba6cff6d0678e2f83d45c1c08425144c Mon Sep 17 00:00:00 2001 From: vllm-agent Date: Thu, 17 Sep 2026 10:02:24 -0700 Subject: [PATCH 045/143] [CI][Bugfix] Add tp_shard_with_padding to padded MoE reload test mock (#57402) Signed-off-by: kimi-no-na-wa Co-authored-by: kimi-no-na-wa Co-authored-by: Misha Goin --- tests/model_executor/model_loader/test_reload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index 844149ce1b..fde8b86cda 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -489,6 +489,7 @@ def test_padded_moe_reload_releases_each_layer( is_act_and_mul=is_gated, has_bias=has_bias, tp_rank=tp_rank, + tp_shard_with_padding=False, moe_parallel_config=SimpleNamespace(tp_size=2), ) model = torch.nn.ModuleList() From f3aa88d2309523f8438a239a55f59a28179ce5ef Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:05:52 -0400 Subject: [PATCH 046/143] [Kimi K3 Bug] Fix kimi k3 reasoning parser (#57098) Signed-off-by: yewentao256 Co-authored-by: Flora Feng <4florafeng@gmail.com> --- vllm/reasoning/kimi_k3_reasoning_parser.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/vllm/reasoning/kimi_k3_reasoning_parser.py b/vllm/reasoning/kimi_k3_reasoning_parser.py index d3ae2b3abe..e5618d47be 100644 --- a/vllm/reasoning/kimi_k3_reasoning_parser.py +++ b/vllm/reasoning/kimi_k3_reasoning_parser.py @@ -256,11 +256,13 @@ class KimiK3ReasoningParser(ReasoningParser): ) -> tuple[str | None, str | None]: """Split full text into ``(reasoning, rest)`` for the non-streaming path. - Handles three shapes: - * no think channel at all -> ``(None, model_output)`` (all content) + Handles four shapes: + * response opener without think markers -> no think channel; all content * open marker present -> reasoning starts after ``<|open|>think<|sep|>`` * open marker absent but a close marker exists (gen-prefix consumed the open) -> reasoning starts at offset 0 + * neither marker present -> truncated reasoning after a consumed + generation prefix ``rest`` is whatever follows the close marker, fed on to the tool parser. """ if not self._thinking_enabled: @@ -270,11 +272,14 @@ class KimiK3ReasoningParser(ReasoningParser): # reasoning content begins right after think-open (or at start if the # open marker was already consumed as a generation prefix) content_start = m_open.end() if m_open is not None else 0 - # if there is no think channel at all, everything is content - if m_open is None and self._think_close_re.search(model_output) is None: - return None, self._content_after_reasoning(model_output, request) m_close = self._think_close_re.search(model_output, content_start) + if ( + m_open is None + and m_close is None + and self._response_open_re.search(model_output) is not None + ): + return None, self._content_after_reasoning(model_output, request) if m_close is not None: reasoning = model_output[content_start : m_close.start()] rest = model_output[m_close.end() :] From ff6b5808c4571a37ba3d27f74d74ef43c3a14245 Mon Sep 17 00:00:00 2001 From: Sriram Kumar Date: Thu, 17 Sep 2026 20:51:35 +0300 Subject: [PATCH 047/143] [ROCm] Resolve the indexer fp8 cache dtype once at import (#53792) Signed-off-by: Sriram Kumar Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> --- .../v1/attention/ops/rocm_aiter_mla_sparse.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 07cdf18106..3d891c8ce0 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -29,6 +29,8 @@ else: logger = init_logger(__name__) +FP8_DTYPE = current_platform.fp8_dtype() + @functools.cache def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None: @@ -260,8 +262,7 @@ def indexer_k_quant_and_cache_triton( # In real layout, we store the first portion as kv cache value # and second portion as kv cache scale kv_cache = kv_cache.view(num_blocks, -1) - fp8_dtype = current_platform.fp8_dtype() - kv_cache_value = kv_cache[:, : block_size * head_dim].view(fp8_dtype) + kv_cache_value = kv_cache[:, : block_size * head_dim].view(FP8_DTYPE) kv_cache_scale = kv_cache[:, block_size * head_dim :].view(torch.float32) head_tile_size = head_tile_size // kv_cache.element_size() layout = "NORMAL" if block_size == 1 else "SHUFFLE" @@ -279,7 +280,7 @@ def indexer_k_quant_and_cache_triton( layout, block_tile_size, head_tile_size, - IS_FNUZ=current_platform.fp8_dtype() == torch.float8_e4m3fnuz, + IS_FNUZ=torch.float8_e4m3fnuz == FP8_DTYPE, USE_UE8M0=scale_fmt == "ue8m0", ) @@ -465,8 +466,7 @@ def cp_gather_indexer_k_quant_cache_triton( num_blocks = k_cache.shape[0] # we assume the kv cache already been split to 2 portion k_cache = k_cache.view(num_blocks, -1) - fp8_dtype = current_platform.fp8_dtype() - k_cache_value = k_cache[:, : block_size * head_dim].view(fp8_dtype) + k_cache_value = k_cache[:, : block_size * head_dim].view(FP8_DTYPE) k_cache_scale = k_cache[:, block_size * head_dim :].view(torch.float32) grid = (num_tokens,) k_fp8_scale = k_fp8_scale.view(torch.float32) @@ -516,7 +516,6 @@ def fp8_paged_mqa_logits_torch( ): from vllm.utils.math_utils import cdiv - fp8_dtype = current_platform.fp8_dtype() batch_size, next_n, _, dim = q.size() if next_n == 1: block_size = kv_cache.shape[1] @@ -540,7 +539,7 @@ def fp8_paged_mqa_logits_torch( cache = kv_cache_flat[pages] scale_offset = block_size * dim cache_value = ( - cache[..., :scale_offset].view(dtype=fp8_dtype).to(torch.float32) + cache[..., :scale_offset].view(dtype=FP8_DTYPE).to(torch.float32) ) cache_scale = ( cache[..., scale_offset:].view(dtype=torch.float32).contiguous() @@ -558,7 +557,7 @@ def fp8_paged_mqa_logits_torch( kv_cache, scale = kv_cache[..., :dim], kv_cache[..., dim:] scale = scale.contiguous().view(torch.float) q = q.float() - kv_cache = kv_cache.view(fp8_dtype).float() * scale + kv_cache = kv_cache.view(FP8_DTYPE).float() * scale num_block, block_size, _, dim = kv_cache.size() logits = torch.full( [batch_size * next_n, max_model_len], @@ -1021,7 +1020,6 @@ def rocm_aiter_sparse_attn_indexer( # careful! this will be None in dummy run forward_context = get_forward_context() attn_metadata = forward_context.attn_metadata - fp8_dtype = current_platform.fp8_dtype() from vllm.utils.torch_utils import _resolve_layer_name k_cache_prefix = _resolve_layer_name(k_cache_prefix) @@ -1037,7 +1035,7 @@ def rocm_aiter_sparse_attn_indexer( # Prefill k_fp8 and k_scale buffers, used by # rocm_aiter_sparse_attn_indexer's prefill path workspace_manager.get_simultaneous( - ((total_seq_lens, head_dim), fp8_dtype), + ((total_seq_lens, head_dim), FP8_DTYPE), ((total_seq_lens, 4), torch.uint8), ) @@ -1117,7 +1115,7 @@ def rocm_aiter_sparse_attn_indexer( workspace_manager = current_workspace_manager() k_fp8_full, k_scale_full = workspace_manager.get_simultaneous( - ((total_seq_lens, head_dim), fp8_dtype), + ((total_seq_lens, head_dim), FP8_DTYPE), ((total_seq_lens, 4), torch.uint8), ) for chunk in prefill_metadata.chunks: From 09c379ab6183085fd90e8195792cc63f5247cdf3 Mon Sep 17 00:00:00 2001 From: Andy Lo Date: Thu, 17 Sep 2026 19:09:18 +0100 Subject: [PATCH 048/143] [Bugfix] Max-load throughput cliff when `max_num_seqs` is not a multiple of 8 (#57355) Signed-off-by: Andy Lo Co-authored-by: OpenAI Codex --- tests/compile/test_config.py | 19 +++++++++++++++++++ vllm/config/vllm.py | 8 +++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 5c9132ccbf..7465ea442c 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -541,6 +541,25 @@ def _mock_config_for_cudagraph_sizes( return config +@pytest.mark.parametrize("max_num_seqs", [100, 101]) +def test_default_cudagraph_capture_sizes_cover_off_stride_max_num_seqs( + max_num_seqs: int, +) -> None: + compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE + ) + config = _mock_config_for_cudagraph_sizes( + max_num_seqs=max_num_seqs, + num_speculative_tokens=0, + max_num_batched_tokens=32768, + compilation_config=compilation_config, + ) + + VllmConfig._set_cudagraph_sizes(config) + + assert max_num_seqs in compilation_config.cudagraph_capture_sizes + + @pytest.mark.parametrize( ("max_num_seqs", "num_speculative_tokens", "widest_is_captured"), [ diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index e5710dced3..4431743a3d 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2223,9 +2223,9 @@ class VllmConfig: max_cudagraph_capture_size = ( self.compilation_config.max_cudagraph_capture_size ) - # Decode sizes to cover, in tokens. Populated only when a request - # is more than one token wide and only when the default is computed - # here, so an explicit capture range is left exactly as configured. + # Decode sizes to cover, in tokens. Populated only when the default + # is computed here, so an explicit capture range is left exactly as + # configured. uniform_decode_sizes: list[int] = [] if max_cudagraph_capture_size is None: from vllm.platforms import current_platform @@ -2316,6 +2316,8 @@ class VllmConfig: if n * query_len <= max_cudagraph_capture_size } ) + elif max_num_seqs <= max_cudagraph_capture_size: + uniform_decode_sizes = [max_num_seqs] max_num_tokens = self.scheduler_config.max_num_batched_tokens max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size) From 6a344d8c8595fc7ca32a633b6f146d36dd227430 Mon Sep 17 00:00:00 2001 From: Michele Campi <215741962+MicheleCampi@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:19:13 +0200 Subject: [PATCH 049/143] [Bugfix][Metrics] Do not log a 0.0% prefix cache hit rate before any query (#54990) Signed-off-by: Michele Campi <215741962+MicheleCampi@users.noreply.github.com> --- tests/v1/core/test_kv_cache_utils.py | 16 ++++++++++++++++ vllm/v1/metrics/loggers.py | 11 ++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 3c82180474..07c2222145 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1068,6 +1068,22 @@ def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats: return PrefixCacheStats(requests=requests, queries=queries, hits=hits) +def test_metrics_empty_distinguishes_no_queries_from_no_hits(): + """`hit_rate` alone cannot tell the two apart; `empty` can. + + Both an unobserved window and a genuine all-miss window report a hit + rate of 0.0, so anything surfacing that number to a human has to check + `empty` first - which is what the prefix-cache log line does. + """ + metrics = CachingMetrics(max_recent_requests=5) + assert metrics.empty + assert metrics.hit_rate == 0.0 + + metrics.observe(_stats(1, 20, 0)) + assert not metrics.empty + assert metrics.hit_rate == 0.0 + + def test_metrics(): """Test the prefix caching metrics.""" metrics = CachingMetrics(max_recent_requests=5) diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index fee6679fe7..26a4142df9 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -296,19 +296,16 @@ class LoggingStatLogger(StatLoggerBase): log_parts.append("Preemptions: %d") log_args.append(self.num_preemptions) - log_parts.extend( - [ - "%s KV cache usage: %.1f%%", - "Prefix cache hit rate: %.1f%%", - ] - ) + log_parts.append("%s KV cache usage: %.1f%%") log_args.extend( [ self.kv_cache_device, self.last_scheduler_stats.kv_cache_usage * 100, - self.prefix_caching_metrics.hit_rate * 100, ] ) + if not self.prefix_caching_metrics.empty: + log_parts.append("Prefix cache hit rate: %.1f%%") + log_args.append(self.prefix_caching_metrics.hit_rate * 100) if envs.VLLM_COMPUTE_NANS_IN_LOGITS: log_parts.append("Corrupted: %d reqs") From a9a7e45f3174781f2b20c03c7ac1beee2b98f9d5 Mon Sep 17 00:00:00 2001 From: afriedri Date: Thu, 17 Sep 2026 13:39:52 -0500 Subject: [PATCH 050/143] [ROCm] Restore `VLLM_ROCM_USE_AITER_FP4_ASM_GEMM` and default w4a4 ASM GEMM back to off (#57055) Signed-off-by: Andy Friedrich --- tests/kernels/quantization/test_rocm_mxfp4.py | 16 ++++++++++++---- vllm/_aiter_ops.py | 6 +++++- vllm/envs.py | 6 ++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/kernels/quantization/test_rocm_mxfp4.py b/tests/kernels/quantization/test_rocm_mxfp4.py index 0e14e2d258..8d1a4bb0a7 100644 --- a/tests/kernels/quantization/test_rocm_mxfp4.py +++ b/tests/kernels/quantization/test_rocm_mxfp4.py @@ -156,22 +156,26 @@ def test_fp4_env_defaults(): """ROCm FP4 env defaults should stay stable for the AITER gates.""" import vllm.envs as envs + assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False assert envs.VLLM_ROCM_USE_AITER_FP4BMM is True @pytest.mark.parametrize( ( "use_aiter", + "use_fp4_asm_gemm", "use_fp4bmm", ), [ - (True, True), - (True, False), - (False, True), + (True, True, True), + (True, True, False), + (True, False, True), + (False, True, True), ], ) def test_rocm_aiter_fp4_enablement_follows_env_and_arch( use_aiter, + use_fp4_asm_gemm, use_fp4bmm, monkeypatch, ): @@ -187,11 +191,15 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch( _assert_aiter_supported() on_gfx950_value = on_gfx950() - expected_asm_gemm = use_aiter and on_gfx950_value + expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value with monkeypatch.context() as mp: mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0") + mp.setenv( + "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", + "1" if use_fp4_asm_gemm else "0", + ) mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0") _reload_envs() rocm_aiter_ops.refresh_env_variables() diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 4b9a145f48..c514b7c0bf 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1893,6 +1893,7 @@ class rocm_aiter_ops: VLLM_ROCM_USE_AITER_MHA: Controls MHA ops including flash_attn_varlen. VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: Controls Triton unified attention. VLLM_ROCM_USE_AITER_FP8BMM: Controls FP8 batched matrix multiply. + VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4). @@ -1960,6 +1961,8 @@ class rocm_aiter_ops: _FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM _FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM _LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM + # TODO: Consolidate under _LINEAR_ENABLED + _FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM # TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE _TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS @@ -1989,6 +1992,7 @@ class rocm_aiter_ops: cls._FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM cls._FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM cls._LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM + cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 @@ -2230,7 +2234,7 @@ class rocm_aiter_ops: def is_asm_fp4_gemm_dynamic_quant_enabled(cls) -> bool: from vllm.platforms.rocm import on_gfx950 - return cls._AITER_ENABLED and on_gfx950() + return cls._AITER_ENABLED and cls._FP4_GEMM_DYNAMIC_QUANT_ASM and on_gfx950() @classmethod @if_aiter_supported diff --git a/vllm/envs.py b/vllm/envs.py index c512df457a..82d6e771b5 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -143,6 +143,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto" VLLM_ROCM_USE_AITER_MHA: bool = True + VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False VLLM_ROCM_USE_AITER_FP8BMM: bool = True VLLM_ROCM_USE_AITER_FP4BMM: bool = True @@ -1305,6 +1306,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MHA": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MHA", "True").lower() in ("true", "1") ), + # Whether to use aiter fp4 gemm asm. + # By default is disabled. + "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM": lambda: ( + os.getenv("VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", "False").lower() in ("true", "1") + ), # Whether to use aiter rope. # By default is disabled. "VLLM_ROCM_USE_AITER_TRITON_ROPE": lambda: ( From 41f9104fa649aed2c26af8a459210619ad9514fe Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Fri, 18 Sep 2026 03:06:36 +0800 Subject: [PATCH 051/143] [DSv4.1] Integrate Mega-Gate from DeepGEMM (#56266) Signed-off-by: zjy0516 Signed-off-by: Thien Tran Co-authored-by: OpenAI Codex Co-authored-by: zjy0516 --- .buildkite/test_areas/kernels.yaml | 1 + tests/kernels/test_mhc_kernels.py | 5 +- tests/models/test_deepseek_v4_mega_moe.py | 150 ++++++++++++++++++++-- vllm/models/deepseek_v4/nvidia/dspark.py | 14 ++ vllm/models/deepseek_v4/nvidia/model.py | 129 ++++++++++++++++--- vllm/models/deepseek_v4/nvidia/mtp.py | 21 ++- vllm/models/deepseek_v41/nvidia/dspark.py | 14 ++ vllm/models/deepseek_v41/nvidia/model.py | 16 ++- vllm/utils/deep_gemm.py | 54 ++++++++ 9 files changed, 369 insertions(+), 35 deletions(-) diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 5a7832d945..583378fdb4 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -120,6 +120,7 @@ steps: - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu - vllm/models/deepseek_v4/common/ops/ - vllm/models/deepseek_v4/nvidia/ + - vllm/utils/deep_gemm.py - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py - tests/models/test_deepseek_v4_mega_moe.py commands: diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py index fafb37d97b..08b1a24c85 100644 --- a/tests/kernels/test_mhc_kernels.py +++ b/tests/kernels/test_mhc_kernels.py @@ -132,6 +132,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch): monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams) draft = SimpleNamespace( + use_mega_moe=False, use_sequence_parallel=False, hc_mult=hc_mult, layers=[make_layer(mix) for mix in mixes], @@ -452,7 +453,7 @@ def test_deepseek_v41_decoder_mixes_match_torch( device=DEVICE, dtype=torch.bfloat16 ) decoder.attn = lambda positions, x, _: x * 0.5 - decoder.ffn = lambda x, input_ids: x * 0.25 + decoder.ffn = lambda x, input_ids, mega_gate_metadata=None: x * 0.25 with torch.device(DEVICE): decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02 decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02 @@ -536,7 +537,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf device=DEVICE, dtype=torch.bfloat16 ) decoder.attn = lambda positions, x, _: x * 0.5 - decoder.ffn = lambda x, input_ids: x * 0.25 + decoder.ffn = lambda x, input_ids, mega_gate_metadata=None: x * 0.25 with torch.device(DEVICE): decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02 decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02 diff --git a/tests/models/test_deepseek_v4_mega_moe.py b/tests/models/test_deepseek_v4_mega_moe.py index 67029b50a9..ad029160c5 100644 --- a/tests/models/test_deepseek_v4_mega_moe.py +++ b/tests/models/test_deepseek_v4_mega_moe.py @@ -15,6 +15,7 @@ from vllm.models.deepseek_v4.nvidia.model import ( DeepseekV4MegaMoEExperts, DeepseekV4MoE, make_deepseek_v4_expert_params_mapping, + prepare_mega_gate_routing_metadata, ) from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs @@ -22,6 +23,7 @@ from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE from vllm.platforms import current_platform from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config +from vllm.utils.torch_utils import set_default_torch_dtype pytestmark = pytest.mark.skipif( not current_platform.is_cuda(), @@ -33,6 +35,7 @@ pytestmark = pytest.mark.skipif( def v41_moe_config(dist_init): return SimpleNamespace( model_config=SimpleNamespace( + dtype=torch.bfloat16, hf_config=DeepseekV41Config( text_config=dict( hidden_size=128, @@ -63,28 +66,52 @@ def v41_moe_config(dist_init): ) +@pytest.mark.parametrize("use_cudagraph", [False, True]) +@pytest.mark.parametrize("above_threshold", [False, True]) @pytest.mark.parametrize("vision", [False, True]) -@pytest.mark.parametrize("layer_id,num_experts,top_k", [(0, 8, 2), (2, 4, 3)]) +@pytest.mark.parametrize("layer_id,num_experts,top_k", [(0, 384, 6), (2, 128, 3)]) def test_deepseek_v41_moe_routes_without_hash_table( - v41_moe_config, monkeypatch, vision, layer_id, num_experts, top_k + v41_moe_config, + monkeypatch, + vision, + layer_id, + num_experts, + top_k, + above_threshold, + use_cudagraph, ): """Main and draft layers select experts by score and preserve image routing.""" + if not current_platform.is_device_capability_family(100): + pytest.skip("DeepGEMM Mega Gate requires SM100") + config = v41_moe_config.model_config.hf_config + config.hidden_size = 256 + config.n_routed_experts = 384 + config.num_experts_per_tok = 6 + config.dspark_n_routed_experts = 128 + num_tokens = (1 if num_experts == 128 else 16) + int(above_threshold) config.vision_n_layers = int(vision) - with torch.device("cuda"): + with ( + set_default_torch_dtype(v41_moe_config.model_config.dtype), + torch.device("cuda"), + ): moe = DeepseekV41MoE(v41_moe_config, prefix=f"model.layers.{layer_id}.ffn") - hidden_states = torch.randn(4, config.hidden_size) + hidden_states = torch.randn( + num_tokens, config.hidden_size, dtype=torch.bfloat16 + ) input_ids = ( - torch.tensor([42, IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID, 129257]) + torch.tensor( + [42, IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID, 129257] + ).repeat((num_tokens + 3) // 4)[:num_tokens] if vision else None ) assert moe.gate.tid2eid is None assert moe.gate.weight.shape == (num_experts, config.hidden_size) assert isinstance(moe.experts, DeepseekV4MegaMoEExperts) - assert moe.experts.w13_weight.shape == (num_experts, 256, 64) + assert moe.experts.w13_weight.shape == (num_experts, 256, 128) assert moe.experts.top_k == top_k - assert (config.n_routed_experts, config.num_experts_per_tok) == (8, 2) + assert (config.n_routed_experts, config.num_experts_per_tok) == (384, 6) with torch.no_grad(): moe.gate.weight.normal_(std=0.01) @@ -93,7 +120,7 @@ def test_deepseek_v41_moe_routes_without_hash_table( moe.gate.bias_vl.copy_(-moe.gate.e_score_correction_bias) scores = torch.nn.functional.softplus( - torch.nn.functional.linear(hidden_states, moe.gate.weight) + torch.mm(hidden_states, moe.gate.weight.t(), out_dtype=torch.float32) ).sqrt() bias = moe.gate.e_score_correction_bias if vision: @@ -105,15 +132,49 @@ def test_deepseek_v41_moe_routes_without_hash_table( dim=-1, keepdim=True ) + routed = {} + def check_routing(x, weights, ids, *, activation_clamp): - torch.testing.assert_close(ids, expected_ids) - torch.testing.assert_close(weights, expected_weights) + routed["ids"], routed["weights"] = ids, weights assert activation_clamp == config.swiglu_limit return x.clone() monkeypatch.setattr(moe.experts, "forward", check_routing) monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x) - torch.testing.assert_close(moe(hidden_states, input_ids), 3 * hidden_states) + routing_input_ids = ( + input_ids + if input_ids is not None + else torch.zeros(num_tokens, dtype=torch.int64, device="cuda") + ) + metadata = prepare_mega_gate_routing_metadata( + routing_input_ids, + has_hash_routing=False, + image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID if vision else None, + ) + if use_cudagraph: + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + moe(hidden_states, input_ids, metadata) + stream.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + output = moe(hidden_states, input_ids, metadata) + graph.replay() + torch.accelerator.synchronize() + else: + output = moe(hidden_states, input_ids, metadata) + + ids, weights = routed["ids"], routed["weights"] + torch.testing.assert_close(ids, expected_ids) + torch.testing.assert_close( + weights, + expected_weights, + rtol=1e-3, + atol=1e-4, + ) + torch.testing.assert_close(output, 3 * hidden_states) @pytest.mark.parametrize( @@ -156,6 +217,67 @@ def test_deepseek_v4_moe_preserves_configured_hash_layers(v41_moe_config): moe(torch.zeros(1, 128)) +@pytest.mark.parametrize("num_tokens", [16, 17]) +def test_deepseek_v4_mega_gate_hash_routing_correctness( + v41_moe_config, monkeypatch, num_tokens +): + if not current_platform.is_device_capability_family(100): + pytest.skip("DeepGEMM Mega Gate requires SM100") + + config = v41_moe_config.model_config.hf_config + config.hidden_size = 256 + config.num_hash_layers = 1 + config.vocab_size = 32 + with ( + set_default_torch_dtype(v41_moe_config.model_config.dtype), + torch.device("cuda"), + ): + moe = DeepseekV4MoE( + v41_moe_config, + prefix="model.layers.0.ffn", + num_hash_layers=config.num_hash_layers, + ) + hidden_states = torch.randn( + num_tokens, config.hidden_size, dtype=torch.bfloat16 + ) + input_ids = torch.arange(num_tokens) + + token_ids = torch.arange(config.vocab_size, device="cuda") + fixed_ids = torch.stack( + ( + token_ids % config.n_routed_experts, + (token_ids + 3) % config.n_routed_experts, + ), + dim=1, + ) + with torch.no_grad(): + moe.gate.weight.normal_(std=0.01) + moe.gate.tid2eid.copy_(fixed_ids) + + expected_ids = fixed_ids[input_ids] + scores = torch.nn.functional.softplus( + torch.mm(hidden_states, moe.gate.weight.t(), out_dtype=torch.float32) + ).sqrt() + expected_weights = scores.gather(1, expected_ids) + expected_weights *= config.routed_scaling_factor / expected_weights.sum( + dim=-1, keepdim=True + ) + + def check_routing(x, weights, ids, *, activation_clamp): + torch.testing.assert_close(ids, expected_ids) + torch.testing.assert_close(weights, expected_weights, rtol=1e-3, atol=1e-4) + return x.clone() + + monkeypatch.setattr(moe.experts, "forward", check_routing) + monkeypatch.setattr(moe.shared_experts, "forward", torch.zeros_like) + metadata = prepare_mega_gate_routing_metadata( + input_ids, + has_hash_routing=True, + image_sentinel_base_id=None, + ) + torch.testing.assert_close(moe(hidden_states, input_ids, metadata), hidden_states) + + def test_deepseek_v4_mega_moe_expert_mapping(): mapping = make_deepseek_v4_expert_params_mapping(2) @@ -513,6 +635,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert( monkeypatch, fused ): class FakeGate(torch.nn.Module): + weight = torch.empty(2, 128) tid2eid = None e_score_correction_bias = None @@ -760,7 +883,8 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe(): not torch.cuda.is_available(), reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.", ) -def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(): +@pytest.mark.parametrize("nonfinite_padding", [False, True]) +def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(nonfinite_padding): from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8 device = torch.device("cuda") @@ -796,6 +920,8 @@ def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(): [False, True, False, False, True, False, True], device=device, ) + if nonfinite_padding: + topk_weights[is_padding] = float("nan") ref_x, ref_x_sf = per_token_cast_to_fp8( hidden_states, diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index 52783889d6..987909702c 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -17,6 +17,7 @@ import torch.nn as nn import vllm.envs as envs from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config.kernel import MEGA_MOE_BACKENDS from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -51,12 +52,14 @@ from vllm.models.common.ops.sequence_parallel import ( sp_padding_mask, sp_shard, ) +from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from .model import ( DeepseekV4DecoderLayer, DeepseekV4Model, _use_sequence_parallel, make_deepseek_v4_expert_params_mapping, + prepare_mega_gate_routing_metadata, ) logger = init_logger(__name__) @@ -96,6 +99,7 @@ class DSparkDeepseekV4Model(nn.Module): self.rms_norm_eps = config.rms_norm_eps self.num_hidden_layers = config.num_hidden_layers self.target_layer_ids = tuple(config.dspark_target_layer_ids) + self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS self.use_sequence_parallel = _use_sequence_parallel(vllm_config) self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3 @@ -235,6 +239,15 @@ class DSparkDeepseekV4Model(nn.Module): ) inputs_embeds = sp_shard(inputs_embeds) input_ids = sp_shard(input_ids) + mega_gate_metadata = None + if self.use_mega_moe: + mega_gate_metadata = prepare_mega_gate_routing_metadata( + input_ids, + has_hash_routing=False, + image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID + if getattr(self.config, "vision_n_layers", 0) > 0 + else None, + ) # Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]). hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1) @@ -247,6 +260,7 @@ class DSparkDeepseekV4Model(nn.Module): post_mix, res_mix, residual, + mega_gate_metadata=mega_gate_metadata, ) hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix) if self.use_sequence_parallel: diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index 7fef691bd1..59156365c5 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -98,6 +98,37 @@ from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID logger = init_logger(__name__) +class MegaGateRoutingMetadata(typing.NamedTuple): + safe_hash_input_ids: torch.Tensor | None + image_token_mask: torch.Tensor | None + hash_token_mask: torch.Tensor | None + + +def prepare_mega_gate_routing_metadata( + input_ids: torch.Tensor, + *, + has_hash_routing: bool, + image_sentinel_base_id: int | None, +) -> MegaGateRoutingMetadata: + if image_sentinel_base_id is None: + return MegaGateRoutingMetadata( + input_ids if has_hash_routing else None, + None, + torch.ones_like(input_ids, dtype=torch.bool) if has_hash_routing else None, + ) + + image_token_mask = (input_ids >= image_sentinel_base_id) & ( + input_ids < image_sentinel_base_id + 5 + ) + if not has_hash_routing: + return MegaGateRoutingMetadata(None, image_token_mask, None) + return MegaGateRoutingMetadata( + torch.where(image_token_mask, 0, input_ids), + image_token_mask, + ~image_token_mask, + ) + + class DeepseekV4MLP(nn.Module): def __init__( self, @@ -803,6 +834,10 @@ class DeepseekV4MoE(nn.Module): raise NotImplementedError( "DeepSeek V4 MegaMoE currently supports sqrtsoftplus routing only." ) + if self.use_mega_moe and not self.renormalize: + raise NotImplementedError( + "DeepSeek V4 MegaMoE requires normalized top-k probabilities." + ) if self.use_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4": raise NotImplementedError( "DeepSeek V4 MegaMoE only supports fp4 experts; got expert_dtype=" @@ -1000,7 +1035,10 @@ class DeepseekV4MoE(nn.Module): ) def forward( - self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor | None = None, + mega_gate_metadata: MegaGateRoutingMetadata | None = None, ) -> torch.Tensor: if self.gate.tid2eid is not None and input_ids is None: raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.") @@ -1013,23 +1051,61 @@ class DeepseekV4MoE(nn.Module): return self._forward_fused_moe(hidden_states, input_ids) org_shape = hidden_states.shape - router_logits, _ = self.gate(hidden_states) - topk_weights, topk_ids = fused_topk_bias( - hidden_states=hidden_states, - gating_output=router_logits, - scoring_func=self.scoring_func, - e_score_correction_bias=self.gate.e_score_correction_bias.data - if self.gate.e_score_correction_bias is not None - else None, - topk=self.n_activated_experts, - renormalize=self.renormalize, - indices_type=self.hash_indices_dtype, - input_tokens=input_ids, - hash_indices_table=self.gate.tid2eid, - routed_scaling_factor=self.routed_scaling_factor, - bias_vl=bias_vl.data if bias_vl is not None else None, - image_sentinel_lo=self.image_sentinel_lo if bias_vl is not None else 0, - ) + # Small local padded batches favor GateLinear; 128-expert gates cross earlier. + gate_threshold = 1 if self.gate.weight.shape[0] == 128 else 16 + if hidden_states.shape[0] <= gate_threshold: + router_logits, _ = self.gate(hidden_states) + topk_weights, topk_ids = fused_topk_bias( + hidden_states=hidden_states, + gating_output=router_logits, + scoring_func=self.scoring_func, + e_score_correction_bias=self.gate.e_score_correction_bias.data + if self.gate.e_score_correction_bias is not None + else None, + topk=self.n_activated_experts, + renormalize=self.renormalize, + indices_type=self.hash_indices_dtype, + input_tokens=input_ids, + hash_indices_table=self.gate.tid2eid, + routed_scaling_factor=self.routed_scaling_factor, + bias_vl=bias_vl.data if bias_vl is not None else None, + image_sentinel_lo=self.image_sentinel_lo if bias_vl is not None else 0, + ) + else: + from vllm.utils.deep_gemm import bf16_mega_gate + + if mega_gate_metadata is None: + raise RuntimeError( + "MegaMoE routing metadata must be prepared once by the model." + ) + + unmapped_topk_idx = None + fix_routing_mask = None + if self.gate.tid2eid is not None: + if mega_gate_metadata.safe_hash_input_ids is None: + raise RuntimeError("Hash routing requires prepared input IDs.") + if mega_gate_metadata.hash_token_mask is None: + raise RuntimeError("Hash routing requires a prepared routing mask.") + unmapped_topk_idx = self.gate.tid2eid[ + mega_gate_metadata.safe_hash_input_ids + ] + fix_routing_mask = mega_gate_metadata.hash_token_mask + + topk_weights, topk_ids = bf16_mega_gate( + hidden_states, + self.gate.weight, + self.n_activated_experts, + scoring_func=self.scoring_func, + routed_scaling_factor=self.routed_scaling_factor, + ep_rank=self.ep_rank, + bias=self.gate.e_score_correction_bias.data + if self.gate.e_score_correction_bias is not None + else None, + image_bias=bias_vl.data if bias_vl is not None else None, + image_token_mask=mega_gate_metadata.image_token_mask, + fix_routing_mask=fix_routing_mask, + unmapped_topk_idx=unmapped_topk_idx, + ) activation_clamp = ( float(self.swiglu_limit) if self.swiglu_limit is not None else None ) @@ -1248,6 +1324,7 @@ class DeepseekV4DecoderLayer(nn.Module): post_mix: torch.Tensor | None = None, res_mix: torch.Tensor | None = None, residual: torch.Tensor | None = None, + mega_gate_metadata: MegaGateRoutingMetadata | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: attn_norm_weight = self.attn_norm.weight.data attn_norm_eps = self.attn_norm.variance_epsilon @@ -1332,7 +1409,7 @@ class DeepseekV4DecoderLayer(nn.Module): norm_eps=ffn_norm_eps, ) - x = self.ffn(x, input_ids) + x = self.ffn(x, input_ids, mega_gate_metadata) return x, residual, post_mix, res_mix @@ -1394,6 +1471,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): ), prefix=f"{prefix}.layers", ) + self.has_local_hash_moe = self.start_layer < min( + self.end_layer, config.num_hash_layers + ) if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps) @@ -1496,6 +1576,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): hidden_states = sp_shard(hidden_states) input_ids = sp_shard(input_ids) + mega_gate_metadata = None + if self.use_mega_moe: + mega_gate_metadata = prepare_mega_gate_routing_metadata( + input_ids, + has_hash_routing=self.has_local_hash_moe, + image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID + if getattr(self.config, "vision_n_layers", 0) > 0 + else None, + ) + residual, post_mix, res_mix = None, None, None remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors) aux_hidden_states: list[torch.Tensor] = [] @@ -1511,6 +1601,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): post_mix, res_mix, residual, + mega_gate_metadata=mega_gate_metadata, ) if idx + 1 in self.aux_hidden_state_layers: # Reconstruct the aux hidden state for draft models diff --git a/vllm/models/deepseek_v4/nvidia/mtp.py b/vllm/models/deepseek_v4/nvidia/mtp.py index a19cc707b2..50886b0afa 100644 --- a/vllm/models/deepseek_v4/nvidia/mtp.py +++ b/vllm/models/deepseek_v4/nvidia/mtp.py @@ -20,6 +20,7 @@ import torch.nn as nn import vllm.envs as envs from vllm.config import VllmConfig +from vllm.config.kernel import MEGA_MOE_BACKENDS from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -51,6 +52,7 @@ from vllm.models.common.ops.sequence_parallel import ( sp_padding_mask, sp_shard, ) +from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import ( _FUSED_MTP_INPUT_RMSNORM_KERNEL, _MTP_SHARED_HEAD_RMSNORM_KERNEL, @@ -62,6 +64,7 @@ from .model import ( DeepseekV4Model, _use_sequence_parallel, make_deepseek_v4_expert_params_mapping, + prepare_mega_gate_routing_metadata, ) logger = init_logger(__name__) @@ -92,6 +95,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): self.config = config quant_config = vllm_config.quant_config self.rms_norm_eps = config.rms_norm_eps + self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -181,8 +185,23 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): hidden_states = self.h_proj(previous_hidden_states) + self.e_proj( inputs_embeds ).unsqueeze(-2) + mega_gate_metadata = None + if self.use_mega_moe: + routing_input_ids = input_ids + if self.mtp_block.use_sequence_parallel: + routing_input_ids = sp_shard(routing_input_ids) + mega_gate_metadata = prepare_mega_gate_routing_metadata( + routing_input_ids, + has_hash_routing=False, + image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID + if getattr(self.config, "vision_n_layers", 0) > 0 + else None, + ) hidden_states, residual, post_mix, res_mix = self.mtp_block( - positions=positions, x=hidden_states, input_ids=input_ids + positions=positions, + x=hidden_states, + input_ids=input_ids, + mega_gate_metadata=mega_gate_metadata, ) hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix) if self.mtp_block.use_sequence_parallel: diff --git a/vllm/models/deepseek_v41/nvidia/dspark.py b/vllm/models/deepseek_v41/nvidia/dspark.py index 9a07189bc3..c08de12a12 100644 --- a/vllm/models/deepseek_v41/nvidia/dspark.py +++ b/vllm/models/deepseek_v41/nvidia/dspark.py @@ -18,6 +18,7 @@ import torch.nn as nn import vllm.envs as envs from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config.kernel import MEGA_MOE_BACKENDS from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -50,12 +51,14 @@ from vllm.models.common.ops.sequence_parallel import ( sp_shard, ) +from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from .model import ( DeepseekV4DecoderLayer, DeepseekV4Model, _linear_scale_param_name, _use_sequence_parallel, make_deepseek_v4_expert_params_mapping, + prepare_mega_gate_routing_metadata, ) logger = init_logger(__name__) @@ -77,6 +80,7 @@ class DSparkDeepseekV4Model(nn.Module): self.rms_norm_eps = config.rms_norm_eps self.num_hidden_layers = config.num_hidden_layers self.target_layer_ids = tuple(config.dspark_target_layer_ids) + self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS self.use_sequence_parallel = _use_sequence_parallel(vllm_config) self.num_dspark_layers = ( @@ -202,6 +206,15 @@ class DSparkDeepseekV4Model(nn.Module): ) inputs_embeds = sp_shard(inputs_embeds) input_ids = sp_shard(input_ids) + mega_gate_metadata = None + if self.use_mega_moe: + mega_gate_metadata = prepare_mega_gate_routing_metadata( + input_ids, + has_hash_routing=False, + image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID + if getattr(self.config, "vision_n_layers", 0) > 0 + else None, + ) # Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]). hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1) @@ -215,6 +228,7 @@ class DSparkDeepseekV4Model(nn.Module): post_mix, res_mix, residual, + mega_gate_metadata=mega_gate_metadata, ) hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix) # Collapse the hc copies with the pre-mix from the last layer's FFN diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index 21fa00e5c9..6d1faace65 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -61,7 +61,9 @@ from vllm.models.deepseek_v4.nvidia.model import ( DeepseekV4MoE as DeepseekV4MoEBase, ) from vllm.models.deepseek_v4.nvidia.model import ( + MegaGateRoutingMetadata, make_deepseek_v4_expert_params_mapping, + prepare_mega_gate_routing_metadata, ) from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import ( @@ -339,6 +341,7 @@ class DeepseekV4DecoderLayer(nn.Module): engram_mask: torch.Tensor | None = None, *, capture_previous_aux: bool = False, + mega_gate_metadata: MegaGateRoutingMetadata | None = None, ) -> tuple[ torch.Tensor, torch.Tensor, @@ -462,7 +465,7 @@ class DeepseekV4DecoderLayer(nn.Module): norm_weight=self.ffn_norm.weight, norm_eps=self.ffn_norm.variance_epsilon, ) - x = self.ffn(x, input_ids) + x = self.ffn(x, input_ids, mega_gate_metadata) return x, residual, post_mix, res_mix, ffn_pre, previous_aux @@ -705,6 +708,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): hidden_states = sp_shard(hidden_states) input_ids = sp_shard(input_ids) + mega_gate_metadata = None + if self.use_mega_moe: + mega_gate_metadata = prepare_mega_gate_routing_metadata( + input_ids, + has_hash_routing=False, + image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID + if getattr(self.config, "vision_n_layers", 0) > 0 + else None, + ) + residual, post_mix, res_mix = None, None, None pre_mix: torch.Tensor | None = None if not get_pp_group().is_first_rank: @@ -728,6 +741,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): engram_hashes, engram_mask, capture_previous_aux=idx in self.aux_hidden_state_layers, + mega_gate_metadata=mega_gate_metadata, ) if previous_aux is not None: # idx is the one-based id of the layer whose post this is. diff --git a/vllm/utils/deep_gemm.py b/vllm/utils/deep_gemm.py index 3d96b496f1..2a0d4eee4a 100644 --- a/vllm/utils/deep_gemm.py +++ b/vllm/utils/deep_gemm.py @@ -164,6 +164,7 @@ _get_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None _get_paged_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None _tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None _mega_mhc_impl: Callable[..., Any] | None = None +_bf16_mega_gate_impl: Callable[..., Any] | None = None _get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None _get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None _get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = ( @@ -240,6 +241,7 @@ def _lazy_init() -> None: global _get_sparse_mqa_logits_metadata_impl global _get_paged_sparse_mqa_logits_metadata_impl global _tf32_hc_prenorm_gemm_impl, _mega_mhc_impl + global _bf16_mega_gate_impl global _get_mn_major_tma_aligned_tensor_impl global _get_mk_alignment_for_contiguous_layout_impl global _get_theoretical_mk_alignment_for_contiguous_layout_impl @@ -260,6 +262,7 @@ def _lazy_init() -> None: or _get_paged_mqa_logits_metadata_impl is not None or _tf32_hc_prenorm_gemm_impl is not None or _mega_mhc_impl is not None + or _bf16_mega_gate_impl is not None or _get_mk_alignment_for_contiguous_layout_impl is not None or _transform_sf_into_required_layout_impl is not None or _pack_ue8m0_to_int_impl is not None @@ -315,6 +318,7 @@ def _lazy_init() -> None: ) _tf32_hc_prenorm_gemm_impl = getattr(_dg, "tf32_hc_prenorm_gemm", None) _mega_mhc_impl = getattr(_dg, "mega_mhc", None) + _bf16_mega_gate_impl = getattr(_dg, "bf16_mega_gate", None) _get_mn_major_tma_aligned_tensor_impl = getattr( _dg, "get_mn_major_tma_aligned_tensor", None ) @@ -482,6 +486,55 @@ def cublaslt_gemm_nt(*args, **kwargs): return _cublaslt_gemm_nt_impl(*args, **kwargs) +def bf16_mega_gate( + x: torch.Tensor, + weight: torch.Tensor, + num_topk: int, + *, + scoring_func: str, + routed_scaling_factor: float, + ep_rank: int, + bias: torch.Tensor | None = None, + image_bias: torch.Tensor | None = None, + image_token_mask: torch.Tensor | None = None, + fix_routing_mask: torch.Tensor | None = None, + unmapped_topk_idx: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Run DeepGEMM Mega Gate. + + The output order follows vLLM's router contract: weights, then indices. + Passing PyTorch-owned outputs to DeepGEMM makes their storage part of CUDA + graph capture. + """ + _lazy_init() + if _bf16_mega_gate_impl is None: + raise RuntimeError( + "DeepGEMM MegaMoE requires a DeepGEMM build with bf16_mega_gate." + ) + + topk_idx = torch.empty((x.shape[0], num_topk), dtype=torch.int64, device=x.device) + topk_weights = torch.empty( + (x.shape[0], num_topk), dtype=torch.float32, device=x.device + ) + _bf16_mega_gate_impl( + x, + weight, + num_topk, + use_shared_as_routed=False, + num_shared_experts=0, + routed_scaling_factor=routed_scaling_factor, + ep_rank=ep_rank, + scoring_func=scoring_func, + bias=bias, + image_bias=image_bias, + image_token_mask=image_token_mask, + fix_routing_mask=fix_routing_mask, + unmapped_topk_idx=unmapped_topk_idx, + out=(topk_idx, topk_weights), + ) + return topk_weights, topk_idx + + def fp8_gemm_nt(*args, **kwargs): _lazy_init() if _fp8_gemm_nt_impl is None: @@ -988,6 +1041,7 @@ def should_use_deepgemm_for_fp8_linear( __all__ = [ "calc_diff", + "bf16_mega_gate", "DeepGemmQuantScaleFMT", "fp8_gemm_nt", "fp8_einsum", From 67e5b0acc9988afc50d019db64ad9da0dadaa15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Thu, 17 Sep 2026 21:26:10 +0200 Subject: [PATCH 052/143] [Bugfix][HiSparse][NIXL] Import full blocks without tail prefill on D (#57049) Signed-off-by: NickLucche Signed-off-by: Lucas Wilkinson Co-authored-by: OpenAI Codex Co-authored-by: Lucas Wilkinson --- tests/v1/core/test_prefix_caching.py | 104 ++++++++++++++++-- tests/v1/worker/test_utils.py | 65 +++++++++++ .../kv_connector/v1/hisparse/worker.py | 55 +++++++-- .../kv_connector/v1/nixl/base_worker.py | 3 +- .../kv_connector/v1/nixl/pull_worker.py | 10 +- vllm/v1/core/single_type_kv_cache_manager.py | 20 ++-- vllm/v1/hisparse/coordinator.py | 68 ++++++++---- vllm/v1/hisparse/types.py | 5 +- 8 files changed, 277 insertions(+), 53 deletions(-) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 01bf73b6e7..229f9c17f9 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -325,7 +325,7 @@ def test_hisparse_reports_when_context_is_fully_resident(): def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): - """Keep indexer-only imports host-backed in the resident block table.""" + """Restore the final host page when completing an indexer-only import.""" manager = make_hisparse_kv_cache_manager( 32, 16, @@ -360,11 +360,15 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks assert len(source) == len(indexer) == len(resident) == 4 assert len(hot) == 2 - # The local prefix is adopted from shadow pages (GPU-resident), while the - # externally imported page stays host-backed until its tail allocation. assert not any(block.is_null for block in resident[:2]) - assert resident[2].is_null + assert not resident[2].is_null assert not resident[3].is_null + coordinator = get_hisparse_coordinator(manager) + assert not coordinator.build_offload_command().page_transfers + coordinator.finish_host_import(resumed.request_id, failed=False) + transfers = coordinator.build_offload_command().page_transfers + assert len(transfers) == 1 and transfers[0].restore + assert transfers[0].resident_block_ids == (resident[2].block_id,) def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix(): @@ -465,7 +469,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv( ) allocated = manager.get_blocks(resumed.request_id).blocks assert len(allocated[0]) == host_blocks - assert allocated[2][1].is_null == (external > 0) + assert allocated[2][1].is_null == (host_blocks > 2) + assert not allocated[2][-1].is_null manager.free(resumed) # A surviving indexer offload must not make absent host KV look computed. @@ -490,6 +495,85 @@ def allocate_external_prefix( ) +@pytest.mark.parametrize("num_tokens", [1, 15, 16, 17, 31, 32, 33, 127]) +def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens): + """Import every real token and restore the page used by last-token replay.""" + from tests.v1.kv_connector.unit.utils import create_vllm_config + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( + NixlPullConnectorScheduler, + ) + + manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) + config = create_vllm_config(block_size=HISPARSE_BLOCK_SIZE) + connector = NixlPullConnectorScheduler(config, "test", manager.kv_cache_config) + request = make_request( + "import", list(range(num_tokens)), HISPARSE_BLOCK_SIZE, sha256 + ) + request.kv_transfer_params = {"do_remote_prefill": True} + count, is_async = connector.get_num_new_matched_tokens(request, 0) + assert count == num_tokens and is_async + assert allocate_external_prefix(manager, request, count) is not None + source, _, resident, _ = manager.get_blocks(request.request_id).blocks + assert len(source) == len(resident) == (num_tokens + 15) // 16 + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null + coordinator = get_hisparse_coordinator(manager) + assert not coordinator.build_offload_command().page_transfers + + coordinator.finish_host_import(request.request_id, failed=False) + transfers = coordinator.build_offload_command().page_transfers + assert len(transfers) == 1 + restore = transfers[0] + assert restore.restore and not restore.after_forward + assert restore.host_block_id == source[-1].block_id + assert restore.resident_block_ids == (resident[-1].block_id,) + request.num_computed_tokens = count + scheduler = SimpleNamespace( + connector=connector, + kv_cache_manager=manager, + failed_recving_kv_req_ids=set(), + finished_recving_kv_req_ids={request.request_id}, + ) + Scheduler._update_waiting_for_remote_kv(scheduler, request) + assert request.num_tokens - request.num_computed_tokens == 1 + assert manager.allocate_slots(request, num_new_tokens=1) is not None + assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)]) + # A pending restore must not expose uninitialized GPU copies to prefix hits. + assert all(resident[-1] not in copies for copies in coordinator.copies.values()) + + coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1}) + assert resident[-1].ref_cnt == 2 + assert all(resident[-1] not in copies for copies in coordinator.copies.values()) + coordinator.update_spills({}, {restore.transfer_id: 1}) + assert resident[-1].ref_cnt == 1 + assert coordinator.request_states[request.request_id].valid_pages == set( + range(num_tokens // HISPARSE_BLOCK_SIZE) + ) + if num_tokens % HISPARSE_BLOCK_SIZE: + assert source[-1].block_hash is None + else: + assert coordinator.copies[source[-1].block_hash] == (resident[-1],) + + +def test_hisparse_aborted_tail_restore_retains_both_endpoints(): + """An abort cannot recycle storage still used by a queued tail restore.""" + manager = make_hisparse_kv_cache_manager(16, 16) + request = make_request("import", list(range(17)), HISPARSE_BLOCK_SIZE, sha256) + assert allocate_external_prefix(manager, request, 17) is not None + source, _, resident, _ = manager.get_blocks(request.request_id).blocks + host_tail, gpu_tail = source[-1], resident[-1] + coordinator = get_hisparse_coordinator(manager) + coordinator.finish_host_import(request.request_id, failed=False) + restore = coordinator.build_offload_command().page_transfers[0] + manager.free(request) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1 + coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1}) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1 + coordinator.update_spills({}, {restore.transfer_id: 1}) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 0 + assert not coordinator.has_pending_work() + + @pytest.mark.parametrize("enable_caching", [False, True]) def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_caching): """Later admissions must not remove a scheduled request's only readable tier.""" @@ -669,7 +753,7 @@ def test_hisparse_host_cow_copy_is_drained_without_a_gpu_pool(): def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages(): """An in-flight host import must reserve its unwritten resident pages.""" manager = make_hisparse_kv_cache_manager( - 11, + 12, 16, transfer_device_cache=True, ) @@ -1127,9 +1211,10 @@ def test_hisparse_external_import_uses_hard_gpu_footprint(): assert len(source) == num_prompt_blocks assert len(indexer) == num_prompt_blocks assert len(resident) == num_prompt_blocks - assert all(block.is_null for block in resident) + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null assert len(hot) == 2 - assert manager.block_pool.get_num_free_blocks() == 1 + assert manager.block_pool.get_num_free_blocks() == 0 def test_hisparse_external_import_survives_capacity_retry(): @@ -1149,7 +1234,8 @@ def test_hisparse_external_import_survives_capacity_retry(): manager.free(first) assert allocate_external_prefix(manager, second, len(tokens)) is not None _, _, resident, hot = manager.get_blocks(second.request_id).blocks - assert all(block.is_null for block in resident) + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null assert len(hot) == 2 diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 55cab287fd..9a345e926c 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -531,6 +531,71 @@ def test_hisparse_eager_mirror_records_transfer_without_page_copy(): worker._enqueue_transfers.assert_not_called() +@pytest.mark.parametrize("is_host_writer", [False, True]) +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_hisparse_tail_restore_preserves_imported_rows( + monkeypatch, request, is_host_writer, device +): + """Restore externally pinned host rows on every rank under sync checking.""" + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA is required for the device copy check") + worker = _make_hisparse_worker() + worker.is_host_writer = is_host_writer + worker.kernel_block_size = 4 + worker.host_caches = tuple( + torch.arange(24, dtype=torch.float32).reshape(12, 2) + layer * 100 + for layer in range(2) + ) + if device == "cuda": + # Match production's cudaHostRegister pool, which torch.is_pinned() + # does not recognize, rather than torch's caching pinned allocator. + host_pool, registered = hisparse_runtime_module.allocate_pinned_host_pool( + sum(cache.nbytes for cache in worker.host_caches) + ) + request.addfinalizer( + lambda: hisparse_runtime_module.release_pinned_state([], [registered]) + ) + host_caches = host_pool.view(torch.float32).view(2, 12, 2).unbind() + for destination, source in zip(host_caches, worker.host_caches): + destination.copy_(source) + worker.host_caches = host_caches + worker.resident_caches = tuple( + torch.full((4, 4, 2), -1.0, device=device) for _ in range(2) + ) + worker.cache_handles = [ + SimpleNamespace( + runtime=SimpleNamespace(resident_source_index=i, eager_host_mirror=True) + ) + for i in range(2) + ] + worker._record_transfer_completion = MagicMock() + stream = torch.cuda.current_stream() if device == "cuda" else MagicMock() + monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream) + transfer = SparseKVPageTransfer(7, 2, (1, 3), False, restore=True) + + restore_pages = worker._restore_pages + if device == "cuda": + import vllm.utils.gpu_sync_debug as gpu_sync_debug + + monkeypatch.setattr(gpu_sync_debug, "_SYNC_CHECK_MODE", "error") + monkeypatch.setattr(gpu_sync_debug, "_sync_check_enabled", False) + gpu_sync_debug.enable_gpu_sync_check() + restore_pages = gpu_sync_debug.with_gpu_sync_check(restore_pages) + restore_pages([transfer]) + + # Appending into unused capacity must preserve every imported prompt row. + for cache, host, block in zip( + worker.resident_caches, worker.host_caches, transfer.resident_block_ids + ): + assert torch.equal(cache[block].cpu(), host[8:12]) + cache[block, 2:] = 999 + assert torch.equal(cache[block, :2].cpu(), host[8:10]) + assert torch.all(cache[0] == -1) + worker._record_transfer_completion.assert_called_once_with( + [transfer], stream=stream + ) + + def test_hisparse_lazy_mirror_copies_transfer_pages(): worker = _make_hisparse_worker() worker.cache_handles = [ diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py index b7ffe9ec97..b223616401 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py @@ -22,6 +22,7 @@ from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, get_tp_group, ) +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import current_stream from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX @@ -414,11 +415,18 @@ class HiSparseConnectorWorker: transfers = ( metadata.command.page_transfers if metadata.command is not None else [] ) + self._restore_pages([transfer for transfer in transfers if transfer.restore]) self._post_forward_transfers = [ - transfer for transfer in transfers if transfer.after_forward + transfer + for transfer in transfers + if not transfer.restore and transfer.after_forward ] self._submit_transfers( - [transfer for transfer in transfers if not transfer.after_forward] + [ + transfer + for transfer in transfers + if not transfer.restore and not transfer.after_forward + ] ) self._pending_invalid_block_ids.extend(metadata.source_block_ids) if request_state_indices is not None: @@ -708,11 +716,17 @@ class HiSparseConnectorWorker: self._submitted_mirror_layers.update(pending_layers) def _record_transfer_completion( - self, transfers: list[SparseKVPageTransfer] + self, + transfers: list[SparseKVPageTransfer], + *, + stream: torch.Stream | None = None, ) -> None: - if not transfers or not self.is_host_writer: + if not transfers: return - stream = self.dma_stream + if stream is None: + if not self.is_host_writer: + return + stream = self.dma_stream assert stream is not None completion_event = torch.Event() completion_event.record(stream) @@ -720,6 +734,30 @@ class HiSparseConnectorWorker: self._pending_transfer_events.append((completion_event, transfer_ids)) self._enqueued_transfer_ids.extend(transfer_ids) + def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None: + """Restore imported tails on every rank before its forward or graph replay.""" + if not transfers: + return + for layer_index, cache in enumerate(self.cache_handles): + source_index = cache.runtime.resident_source_index + source = self.host_caches[layer_index] + destination = self.resident_caches[layer_index] + for transfer in transfers: + host_start = transfer.host_block_id * self.kernel_block_size + resident_block = transfer.resident_block_ids[source_index] + # Each import pays one H2D page copy and enqueue per layer/rank + # instead of tail prefill on D. Restore the prompt rows before + # decode appends to this page; unused rows do not extend the + # logical sequence. + # cudaHostRegister pins this pool outside PyTorch's allocator, + # so the sync checker cannot recognize it via is_pinned(). + with gpu_sync_allowed(): + destination[resident_block].copy_( + source[host_start : host_start + self.kernel_block_size], + non_blocking=True, + ) + self._record_transfer_completion(transfers, stream=current_stream()) + def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None: if self.cache_handles[0].runtime.eager_host_mirror: self._record_transfer_completion(transfers) @@ -733,15 +771,12 @@ class HiSparseConnectorWorker: descriptor_count = len(transfers) * num_layers descriptors = self._acquire_dma_descriptors(descriptor_count) destination_rows = np.fromiter( - ( - transfer.destination_block_id * self.kernel_block_size - for transfer in transfers - ), + (transfer.host_block_id * self.kernel_block_size for transfer in transfers), dtype=np.int64, count=len(transfers), ) source_blocks_by_transfer = np.asarray( - [transfer.source_block_ids for transfer in transfers], dtype=np.int64 + [transfer.resident_block_ids for transfer in transfers], dtype=np.int64 ) if source_blocks_by_transfer.ndim != 2: raise RuntimeError( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 7bcdd2531b..60cf20303c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -3278,7 +3278,8 @@ class NixlBaseConnectorWorker: """Pair an uncached decode suffix with the same prefill regions.""" assert len(decode_block_ids) == len(prefill_block_ids) if not any(decode_block_ids): - return [], prefill_block_ids + empty_regions: list[list[int]] = [[] for _ in decode_block_ids] + return empty_regions, empty_regions.copy() if num_computed_blocks is not None: assert num_remote_blocks is not None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index fea3a00ce2..203c6bcfa6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -168,8 +168,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): local_block_ids = meta.local_physical_block_ids remote_region_groups = self.dst_region_group_ids[engine_id] local_region_groups = self.region_group_ids or remote_region_groups - groups_differ = local_region_groups != remote_region_groups - if groups_differ: + if not local_block_ids: + # Region expansion cannot index empty groups. Pass empty specs to + # _read_blocks so its existing cache-hit notification path runs. + read_specs = [ + ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[]) + for rank in plan.all_source_ranks + ] + elif local_region_groups != remote_region_groups: if not self.use_mla or self._has_mamba: raise NotImplementedError( "Different NIXL cache-group layouts are only supported for " diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 9ad0a38ed5..835a434c1e 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -2498,12 +2498,14 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager): del num_tokens_main_model assert not new_computed_blocks if total_computed_tokens > num_local_computed_tokens: - if total_computed_tokens % self.block_size != 0: - raise ValueError( - "A host-only HiSparse import must end on a cache-block boundary." - ) - imported_pages = total_computed_tokens // self.block_size - return max(cdiv(num_tokens, self.block_size) - imported_pages, 0) + # Keep the last imported page writable, including the last-token + # replay on a full prompt hit. Only earlier pages stay host-only. + imported_pages = cdiv(total_computed_tokens, self.block_size) + existing = len(self.req_to_blocks.get(request_id, ())) + needs_tail = self.get_resident_page(request_id, imported_pages - 1) is None + return int(needs_tail) + max( + cdiv(num_tokens, self.block_size) - max(imported_pages, existing), 0 + ) existing = len(self.req_to_blocks.get(request_id, ())) host_pages = cdiv(num_local_computed_tokens, self.block_size) required = cdiv(num_tokens, self.block_size) @@ -2518,13 +2520,15 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager): num_local_computed_tokens: int, num_external_computed_tokens: int, ) -> None: - """Represent imported host history with null resident pages.""" + """Reserve a writable final page; earlier imported pages stay on host.""" assert num_external_computed_tokens > 0 num_tokens = num_local_computed_tokens + num_external_computed_tokens blocks = self.req_to_blocks[request_id] - tail_page = cdiv(num_tokens, self.block_size) - 1 + tail_page = (num_tokens - 1) // self.block_size if len(blocks) <= tail_page: blocks.extend([self._null_block] * (tail_page + 1 - len(blocks))) + if blocks[tail_page].is_null: + blocks[tail_page] = self.block_pool.get_new_blocks(1)[0] def add_local_computed_blocks( self, diff --git a/vllm/v1/hisparse/coordinator.py b/vllm/v1/hisparse/coordinator.py index ecbc0a270a..056e19479a 100644 --- a/vllm/v1/hisparse/coordinator.py +++ b/vllm/v1/hisparse/coordinator.py @@ -73,6 +73,7 @@ class _PendingSpill: request_state: _HiSparseRequestState host_block: KVCacheBlock resident_blocks: tuple[KVCacheBlock, ...] + restore: bool = False expected_worker_completions: int = 0 worker_completions: int = 0 enqueue_applied: bool = False @@ -271,25 +272,29 @@ class HiSparseCoordinator: num_computed_tokens // self.host_manager.block_size, len(host_blocks) ) for host_idx in range(state.copies_recorded_blocks, num_host_blocks): - host_block = host_blocks[host_idx] - if host_block.is_null or host_block.block_hash is None: + if host_idx in state.pending_pages: continue - blocks: list[KVCacheBlock] = [] - for manager in self.resident_managers: - block = manager.get_resident_page(request_id, host_idx) - if block is None: - break - blocks.append(block) - if len(blocks) != len(self.resident_managers): - continue - self._drop_copy(host_block.block_hash) - self.copies[host_block.block_hash] = tuple(blocks) - for block in blocks: - self._copy_by_block[block.block_id] = host_block.block_hash + self._record_copy(request_id, host_idx, host_blocks[host_idx]) state.copies_recorded_blocks = max( state.copies_recorded_blocks, num_host_blocks ) + def _record_copy( + self, request_id: str, page_idx: int, host_block: KVCacheBlock + ) -> None: + if host_block.is_null or host_block.block_hash is None: + return + blocks: list[KVCacheBlock] = [] + for manager in self.resident_managers: + block = manager.get_resident_page(request_id, page_idx) + if block is None: + return + blocks.append(block) + self._drop_copy(host_block.block_hash) + self.copies[host_block.block_hash] = tuple(blocks) + for block in blocks: + self._copy_by_block[block.block_id] = host_block.block_hash + def _drop_copy(self, host_hash: BlockHashWithGroupId) -> None: blocks = self.copies.pop(host_hash, None) if blocks is None: @@ -406,9 +411,14 @@ class HiSparseCoordinator: assert self.host_manager is not None host_block_size = self.host_manager.block_size num_pages = num_computed_tokens // host_block_size + importing_pages = cdiv( + self._pending_imports.get(request_id, 0), host_block_size + ) state = self._get_request_state(request_id) budget = max(self.max_spill_pages - len(self.spills_to_send), 0) for page_idx in range(num_pages): + if page_idx < importing_pages: + continue if budget == 0: break if page_idx in state.pending_pages: @@ -420,7 +430,7 @@ class HiSparseCoordinator: for manager in self.resident_managers ): continue - if self._plan_spill(request_id, page_idx, after_forward=True): + if self._plan_page_transfer(request_id, page_idx, after_forward=True): budget -= 1 def publish_when_ready( @@ -491,17 +501,27 @@ class HiSparseCoordinator: state = self._get_request_state(request_id) state.valid_pages = set(range(num_pages)) state.ready_prefix_pages = num_pages + if num_computed_tokens: + self._plan_page_transfer( + request_id, + (num_computed_tokens - 1) // block_size, + after_forward=False, + restore=True, + ) self._publish_host_blocks_if_ready(request_id) - def _plan_spill( + def _plan_page_transfer( self, request_id: str, page_idx: int, *, after_forward: bool, + restore: bool = False, ) -> bool: assert self.host_manager is not None state = self._get_request_state(request_id) + if page_idx in state.pending_pages: + return False host_blocks = self.host_manager.req_to_blocks.get(request_id) if host_blocks is None or page_idx >= len(host_blocks): return False @@ -523,9 +543,10 @@ class HiSparseCoordinator: self.next_spill_id += 1 plan = SparseKVPageTransfer( transfer_id=spill_id, - destination_block_id=host_block.block_id, - source_block_ids=tuple(block.block_id for block in blocks), + host_block_id=host_block.block_id, + resident_block_ids=tuple(block.block_id for block in blocks), after_forward=after_forward, + restore=restore, ) self.pending_spills[spill_id] = _PendingSpill( transfer_id=spill_id, @@ -533,6 +554,7 @@ class HiSparseCoordinator: request_state=state, host_block=host_block, resident_blocks=tuple(blocks), + restore=restore, ) state.pending_pages[page_idx] = spill_id self.spills_to_send.append(plan) @@ -666,7 +688,7 @@ class HiSparseCoordinator: self._complete_host_writes() def _apply_enqueued_spills(self) -> None: - """Drop the DMA pins once the worker has enqueued a spill.""" + """Drop the GPU pins once every worker has completed the transfer.""" for pending in self.pending_spills.values(): if ( pending.enqueue_applied @@ -692,8 +714,12 @@ class HiSparseCoordinator: request_id, page_idx = pending.page pending.request_state.pending_pages.pop(page_idx, None) if self.request_states.get(request_id) is pending.request_state: - pending.request_state.valid_pages.add(page_idx) - self._page_became_clean(request_id, pending.request_state, page_idx) + if not pending.restore: + pending.request_state.valid_pages.add(page_idx) + if page_idx in pending.request_state.valid_pages: + self._page_became_clean(request_id, pending.request_state, page_idx) + if pending.restore: + self._record_copy(request_id, page_idx, pending.host_block) completed_request_ids.add(request_id) assert self.host_manager is not None self.host_manager.block_pool.free_blocks([pending.host_block]) diff --git a/vllm/v1/hisparse/types.py b/vllm/v1/hisparse/types.py index 8618f3cb20..269793ea85 100644 --- a/vllm/v1/hisparse/types.py +++ b/vllm/v1/hisparse/types.py @@ -9,9 +9,10 @@ class SparseKVPageTransfer: """Copy one logical KV page between cache-manager and worker-owned tiers.""" transfer_id: int - destination_block_id: int - source_block_ids: tuple[int, ...] + host_block_id: int + resident_block_ids: tuple[int, ...] after_forward: bool + restore: bool = False @dataclass(frozen=True) From ac2f0ea82c0d3005eb1de081cda715f0ee524c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elvir=20Crn=C4=8Devi=C4=87?= Date: Thu, 17 Sep 2026 21:36:19 +0200 Subject: [PATCH 053/143] [MoE][Bugfix] Skip SP padded rows in grouped MoE routing (#56079) Signed-off-by: Elvir Crncevic Co-authored-by: OpenAI Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Tyler Michael Smith --- tests/kernels/moe/test_grouped_topk.py | 32 +++++++ tests/kernels/moe/test_routing.py | 15 ++++ tests/v1/worker/test_gpu_model_runner.py | 14 +++ tests/v1/worker/test_gpu_model_runner_v2.py | 14 +++ tests/v1/worker/test_gpu_ubatch_slicing.py | 90 +++++++++++++++++++ vllm/model_executor/layers/fused_moe/layer.py | 3 + .../fused_moe/router/grouped_topk_router.py | 14 +++ .../layers/fused_moe/router/router_factory.py | 3 + vllm/models/kimi_k3/nvidia/model.py | 1 + vllm/v1/worker/gpu/model_runner.py | 19 ++-- vllm/v1/worker/gpu_model_runner.py | 16 ++++ vllm/v1/worker/gpu_ubatch_wrapper.py | 9 ++ 12 files changed, 224 insertions(+), 6 deletions(-) diff --git a/tests/kernels/moe/test_grouped_topk.py b/tests/kernels/moe/test_grouped_topk.py index c65cd381db..2004feae65 100644 --- a/tests/kernels/moe/test_grouped_topk.py +++ b/tests/kernels/moe/test_grouped_topk.py @@ -15,8 +15,10 @@ from vllm.config import ( get_cached_compilation_config, set_current_vllm_config, ) +from vllm.forward_context import set_forward_context from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import ( GroupedTopk, + GroupedTopKRouter, fused_grouped_topk, ) from vllm.platforms import current_platform @@ -70,6 +72,36 @@ def _single_group_reference( return values, indices.to(torch.int32) +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform." +) +@pytest.mark.parametrize("skip_padding", [False, True]) +def test_kimi_grouped_topk_masks_padding_when_enabled(skip_padding: bool): + hidden_states = torch.randn((2, 16), device="cuda") + gating_output = torch.randn((2, 896), device="cuda") + is_padding = torch.tensor([False, True], device="cuda") + router = GroupedTopKRouter( + top_k=16, + global_num_experts=896, + num_expert_group=1, + topk_group=1, + skip_padding=skip_padding, + ) + + with set_forward_context(None, VllmConfig(), is_padding=is_padding): + topk_weights, topk_ids = router._compute_routing( + hidden_states, gating_output, torch.int32 + ) + + assert torch.all(topk_ids[0] >= 0) + if skip_padding: + assert torch.all(topk_ids[1] == -1) + assert torch.all(topk_weights[1] == 0) + else: + assert torch.all(topk_ids[1] >= 0) + assert torch.all(topk_weights[1] > 0) + + @pytest.mark.skipif( not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform." ) diff --git a/tests/kernels/moe/test_routing.py b/tests/kernels/moe/test_routing.py index 174950e06c..4baad24cc3 100644 --- a/tests/kernels/moe/test_routing.py +++ b/tests/kernels/moe/test_routing.py @@ -87,6 +87,21 @@ def test_multiple_expert_groups_use_grouped_topk() -> None: ) assert isinstance(router, GroupedTopKRouter) + assert not router.skip_padding + + +def test_grouped_topk_padding_skip_must_be_enabled() -> None: + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + use_grouped_topk=True, + num_expert_group=8, + topk_group=4, + skip_padding=True, + ) + + assert isinstance(router, GroupedTopKRouter) + assert router.skip_padding def test_degenerate_grouped_config_with_bias_uses_topk_bias() -> None: diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 80e8318fa1..32341ce944 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -174,6 +174,20 @@ def test_freeze_gc_disables_and_restores_automatic_gc( gc.disable() +def test_prepare_padding_mask_marks_sequence_parallel_padding(): + runner = GPUModelRunner.__new__(GPUModelRunner) + runner.is_padding = torch.empty(8, dtype=torch.bool) + + mask = runner._prepare_padding_mask(1, 8) + + assert mask.tolist() == [False, True, True, True, True, True, True, True] + assert mask.data_ptr() == runner.is_padding.data_ptr() + + mask = runner._prepare_padding_mask(0, 8) + + assert mask.all() + + @pytest.fixture def model_runner(): vllm_config = get_vllm_config() diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 308c9fe9ea..83cd1e9d07 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -21,6 +21,20 @@ from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.model_runner import GPUModelRunner +def test_prepare_padding_mask_marks_sequence_parallel_padding(): + runner = GPUModelRunner.__new__(GPUModelRunner) + runner.input_buffers = SimpleNamespace(is_padding=torch.empty(8, dtype=torch.bool)) + + mask = runner._prepare_padding_mask(1, 8) + + assert mask.tolist() == [False, True, True, True, True, True, True, True] + assert mask.data_ptr() == runner.input_buffers.is_padding.data_ptr() + + mask = runner._prepare_padding_mask(0, 8) + + assert mask.all() + + def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch): runner = GPUModelRunner.__new__(GPUModelRunner) runner.max_model_len = 262144 diff --git a/tests/v1/worker/test_gpu_ubatch_slicing.py b/tests/v1/worker/test_gpu_ubatch_slicing.py index e735953d8b..9d3a92e623 100644 --- a/tests/v1/worker/test_gpu_ubatch_slicing.py +++ b/tests/v1/worker/test_gpu_ubatch_slicing.py @@ -10,6 +10,7 @@ threaded execution of the microbatches. """ import threading +from contextlib import nullcontext from dataclasses import replace from types import SimpleNamespace from typing import Any, cast @@ -30,6 +31,7 @@ from vllm.config import ( from vllm.forward_context import create_forward_context from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker import gpu_ubatch_wrapper as legacy_gpu_ubatch_wrapper from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils from vllm.v1.worker.gpu import cudagraph_utils, dp_utils from vllm.v1.worker.gpu.cudagraph_utils import ( @@ -426,6 +428,94 @@ def test_slice_model_inputs_handles_mrope_positions(): assert sliced["intermediate_tensors"] is None +@pytest.mark.parametrize( + ("cudagraph_runtime_mode", "execution_method"), + [ + (CUDAGraphMode.FULL, "_capture_ubatches"), + (CUDAGraphMode.NONE, "_run_ubatches"), + ], +) +def test_legacy_ubatch_contexts_slice_padding_mask( + cudagraph_runtime_mode: CUDAGraphMode, execution_method: str +): + """Replacement contexts preserve each microbatch's padding rows.""" + ubatch_slices = [ + UBatchSlice(slice(0, 2), slice(0, 3)), + UBatchSlice(slice(2, 4), slice(3, 6)), + ] + is_padding = torch.tensor([False, True, False, True, False, True]) + forward_context = SimpleNamespace( + batch_descriptor=object(), + ubatch_slices=ubatch_slices, + cudagraph_runtime_mode=cudagraph_runtime_mode, + attn_metadata=[{}, {}], + slot_mapping={}, + dp_metadata=object(), + is_padding=is_padding, + ) + + wrapper = object.__new__(legacy_gpu_ubatch_wrapper.UBatchWrapper) + wrapper.vllm_config = SimpleNamespace( + parallel_config=SimpleNamespace(data_parallel_size=2) + ) + wrapper.comm_stream = object() + wrapper.ready_barrier = object() + wrapper.cudagraphs = {} + wrapper.sm_control = nullcontext() + wrapper.runnable = object() + + def make_contexts(*, forward_contexts, **kwargs): + return [ + SimpleNamespace(id=i, forward_context=context) + for i, context in enumerate(forward_contexts) + ] + + def create_context(attn_metadata, vllm_config, **kwargs): + return SimpleNamespace(attn_metadata=attn_metadata, **kwargs) + + model_inputs = { + "input_ids": torch.arange(6), + "positions": torch.arange(6), + "inputs_embeds": None, + "intermediate_tensors": None, + } + expected = object() + with ( + patch.object( + legacy_gpu_ubatch_wrapper, + "get_forward_context", + return_value=forward_context, + ), + patch.object( + legacy_gpu_ubatch_wrapper.torch.cuda, + "current_stream", + return_value=object(), + ), + patch.object( + legacy_gpu_ubatch_wrapper.DPMetadata, "make", return_value=object() + ), + patch.object( + legacy_gpu_ubatch_wrapper, + "create_forward_context", + side_effect=create_context, + ), + patch.object( + legacy_gpu_ubatch_wrapper, + "make_ubatch_contexts", + side_effect=make_contexts, + ), + patch.object(wrapper, execution_method, return_value=expected) as execute, + ): + result = wrapper(**model_inputs) + + assert result is expected + contexts = [ + metadata.context.forward_context for metadata in execute.call_args.args[0] + ] + torch.testing.assert_close(contexts[0].is_padding, is_padding[:3]) + torch.testing.assert_close(contexts[1].is_padding, is_padding[3:]) + + def _make_dbo_config() -> VllmConfig: return VllmConfig( model_config=ModelConfig(model="facebook/opt-125m", dtype="float16", seed=0), diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 2ee3c5f374..49557bb3f9 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -137,6 +137,7 @@ def FusedMoEFactory( runner_args: dict[str, Any] | None = None, routed_experts_cls: type[RoutedExperts] | None = None, routed_experts_args: dict[str, Any] | None = None, + skip_padding: bool = False, ) -> MoERunner: """Factory function for creating MoE execution pipeline. @@ -213,6 +214,7 @@ def FusedMoEFactory( runner_args: Additional arguments for runner constructor routed_experts_cls: Custom RoutedExperts class (None = use default) routed_experts_args: Additional arguments for routed_experts constructor + skip_padding: Whether grouped routing should invalidate padding rows. Returns: MoERunner: Configured MoE execution pipeline ready for forward passes @@ -308,6 +310,7 @@ def FusedMoEFactory( else 1.0, e_score_correction_bias=e_score_correction_bias, num_fused_shared_experts=num_fused_shared_experts, + skip_padding=(skip_padding and moe_parallel_config.use_deepep_v2_kernels), # Fused shared-expert slot weight. With apply_routed_scale_to_output # the runner scales the combined output by routed_scaling_factor, so # the shared slot weight must be 1/routed_scaling_factor for its net diff --git a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py index 2df3489358..9b134d385a 100644 --- a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py +++ b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py @@ -8,6 +8,7 @@ from vllm import _custom_ops as ops from vllm import envs as envs from vllm._aiter_ops import rocm_aiter_ops from vllm.distributed.eplb.eplb_state import EplbLayerState +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.layers.fused_moe.config import ( RoutingMethodType, @@ -258,6 +259,7 @@ class GroupedTopKRouter(BaseRouter): e_score_correction_bias: torch.Tensor | None = None, num_fused_shared_experts: int = 0, eplb_state: EplbLayerState | None = None, + skip_padding: bool = False, ): super().__init__( top_k=top_k, @@ -271,6 +273,7 @@ class GroupedTopKRouter(BaseRouter): self.routed_scaling_factor = routed_scaling_factor self.e_score_correction_bias = e_score_correction_bias self.num_fused_shared_experts = num_fused_shared_experts + self.skip_padding = skip_padding @property def routing_method_type(self) -> RoutingMethodType: @@ -346,4 +349,15 @@ class GroupedTopKRouter(BaseRouter): e_score_correction_bias=self.e_score_correction_bias, ) + if ( + self.skip_padding + and envs.VLLM_MOE_SKIP_PADDING + and is_forward_context_available() + ): + is_padding = get_forward_context().is_padding + if is_padding is not None: + is_padding = is_padding[: topk_ids.shape[0]].unsqueeze(1) + topk_weights = topk_weights.masked_fill(is_padding, 0) + topk_ids = topk_ids.masked_fill(is_padding, -1) + return topk_weights, topk_ids diff --git a/vllm/model_executor/layers/fused_moe/router/router_factory.py b/vllm/model_executor/layers/fused_moe/router/router_factory.py index 2037251824..f03fb106f7 100644 --- a/vllm/model_executor/layers/fused_moe/router/router_factory.py +++ b/vllm/model_executor/layers/fused_moe/router/router_factory.py @@ -65,6 +65,7 @@ def create_fused_moe_router( # Deepseek V4 vision routing bias parameters bias_vl: torch.Tensor | None = None, image_sentinel_lo: int = 0, + skip_padding: bool = False, ) -> FusedMoERouter: """Factory function to create the appropriate FusedMoERouter subclass based on the provided parameters. @@ -91,6 +92,7 @@ def create_fused_moe_router( topk_group: Top-k within each group (for grouped routing) scoring_func: Scoring function to use ("softmax" or "sigmoid") num_fused_shared_experts: Number of fused shared experts (for ROCm AITER) + skip_padding: Whether to invalidate routes for padding rows Grouped topk and fused topk bias arguments: routed_scaling_factor: Scaling factor for routed weights @@ -209,6 +211,7 @@ def create_fused_moe_router( routed_scaling_factor=routed_scaling_factor, e_score_correction_bias=e_score_correction_bias, num_fused_shared_experts=num_fused_shared_experts, + skip_padding=skip_padding, ) # Otherwise fall through to the non-grouped chain below. diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index 7fe612b654..a970479da8 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -761,6 +761,7 @@ class KimiMoE(nn.Module): routed_input_transform=None, routed_output_transform=self.routed_output_transform, is_sequence_parallel=use_sequence_parallel, + skip_padding=True, runner_cls=LatentMoERunner if self.use_latent_moe else None, ) if self.padded_moe_intermediate_size != moe_intermediate_size: diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 631e6b04d4..903e5abd67 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1258,6 +1258,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_reqs, num_toks, max_query_len, batch_state.has_prefill ) + def _prepare_padding_mask( + self, num_tokens: int, num_tokens_after_padding: int + ) -> torch.Tensor: + is_padding = self.input_buffers.is_padding[:num_tokens_after_padding] + is_padding[:num_tokens].fill_(False) + is_padding[num_tokens:].fill_(True) + return is_padding + def prepare_inputs( self, scheduler_output: SchedulerOutput, @@ -1267,12 +1275,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_tokens = batch_req_state.num_tokens num_tokens_after_padding = max(num_tokens, batch_desc.num_tokens) assert num_tokens > 0 + is_padding = self.input_buffers.is_padding[:num_tokens_after_padding] if envs.VLLM_MOE_SKIP_PADDING: - # Mark trailing cudagraph-padding rows so kernels can skip work for - # them when supported. - is_padding = self.input_buffers.is_padding - is_padding[:num_tokens].fill_(False) - is_padding[num_tokens:num_tokens_after_padding].fill_(True) + is_padding = self._prepare_padding_mask( + num_tokens, num_tokens_after_padding + ) req_ids = batch_req_state.req_ids num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens @@ -1440,7 +1447,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): has_prefill=batch_req_state.has_prefill, input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding], - is_padding=self.input_buffers.is_padding[:num_tokens_after_padding], + is_padding=is_padding, logits_indices=logits_indices, cu_num_logits=cu_num_logits, cu_num_logits_np=cu_num_logits_np, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 77d11621c2..c943f02212 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -816,6 +816,9 @@ class GPUModelRunner( self.positions = torch.zeros( self.max_num_tokens, dtype=torch.int64, device=self.device ) + self.is_padding = torch.zeros( + self.max_num_tokens, dtype=torch.bool, device=self.device + ) self.query_start_loc = self._make_buffer( self.max_num_reqs + 1, dtype=torch.int32 ) @@ -3500,6 +3503,14 @@ class GPUModelRunner( return round_up(num_scheduled_tokens, tp_size) return num_scheduled_tokens + def _prepare_padding_mask( + self, num_tokens_unpadded: int, num_tokens_padded: int + ) -> torch.Tensor: + padding_mask = self.is_padding[:num_tokens_padded] + padding_mask[:num_tokens_unpadded].fill_(False) + padding_mask[num_tokens_unpadded:].fill_(True) + return padding_mask + def _prepare_mm_inputs( self, num_tokens: int ) -> tuple[torch.Tensor | None, torch.Tensor]: @@ -4441,6 +4452,7 @@ class GPUModelRunner( num_tokens_unpadded, ubatch_slices_padded, ) + is_padding = self._prepare_padding_mask(num_tokens_unpadded, num_tokens_padded) with ( set_forward_context( attn_metadata, @@ -4452,6 +4464,7 @@ class GPUModelRunner( ubatch_slices=ubatch_slices_padded, slot_mapping=slot_mappings, skip_compiled=has_encoder_input, + is_padding=is_padding, ), record_function_or_nullcontext("gpu_model_runner: forward"), self.maybe_get_kv_connector_output( @@ -6159,6 +6172,8 @@ class GPUModelRunner( if num_tokens_across_dp is not None: num_tokens_across_dp[:] = num_tokens_padded + is_padding = self._prepare_padding_mask(0, num_tokens_padded) + with ( self.maybe_randomize_inputs( input_ids, inputs_embeds, randomize_inputs=randomize_inputs @@ -6172,6 +6187,7 @@ class GPUModelRunner( batch_descriptor=batch_desc, ubatch_slices=ubatch_slices_padded, slot_mapping=slot_mappings, + is_padding=is_padding, ), ): outputs = self.model( diff --git a/vllm/v1/worker/gpu_ubatch_wrapper.py b/vllm/v1/worker/gpu_ubatch_wrapper.py index a0f2643052..f2cab7898b 100644 --- a/vllm/v1/worker/gpu_ubatch_wrapper.py +++ b/vllm/v1/worker/gpu_ubatch_wrapper.py @@ -261,6 +261,7 @@ class UBatchWrapper: dp_metadata, batch_descriptor, cudagraph_runtime_mode, + is_padding, ) -> list[UbatchMetadata]: # Create one forward context per ubatch forward_contexts = [] @@ -276,6 +277,11 @@ class UBatchWrapper: batch_descriptor=batch_descriptor, cudagraph_runtime_mode=cudagraph_runtime_mode, slot_mapping=slot_mapping[i] if has_slot_mapping else None, + is_padding=( + is_padding[ubatch_slice.token_slice] + if is_padding is not None + else None + ), ) ) @@ -373,6 +379,7 @@ class UBatchWrapper: attn_metadata = forward_context.attn_metadata slot_mapping = forward_context.slot_mapping + is_padding = forward_context.is_padding num_tokens = sum(ubatch_slice.num_tokens for ubatch_slice in ubatch_slices) input_ids = kwargs["input_ids"] positions = kwargs["positions"] @@ -414,6 +421,7 @@ class UBatchWrapper: dp_metadata=ubatch_dp_metadata, batch_descriptor=batch_descriptor, cudagraph_runtime_mode=CUDAGraphMode.NONE, + is_padding=is_padding, ) with self.sm_control: return self._capture_ubatches(ubatch_metadata, self.runnable) @@ -440,6 +448,7 @@ class UBatchWrapper: dp_metadata=ubatch_dp_metadata, batch_descriptor=batch_descriptor, cudagraph_runtime_mode=CUDAGraphMode.NONE, + is_padding=is_padding, ) with self.sm_control: return self._run_ubatches(ubatch_metadata, self.runnable) From 9a5bd373cfe59e236f2b32c30530f4dbbf24120d Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Thu, 17 Sep 2026 16:04:03 -0400 Subject: [PATCH 054/143] [CI] Retire Weight Loading smoke tests (#57398) Signed-off-by: mgoin --- .buildkite/test-amd.yaml | 60 ------------------- .buildkite/test_areas/weight_loading.yaml | 38 ------------ .github/CODEOWNERS | 1 - .../test_rocm_compressed_tensors_w4a16.py | 1 - tests/weight_loading/models-amd.txt | 18 ------ tests/weight_loading/models-large-amd.txt | 4 -- tests/weight_loading/models-large.txt | 6 -- tests/weight_loading/models.txt | 18 ------ .../run_model_weight_loading_test.sh | 58 ------------------ tests/weight_loading/test_weight_loading.py | 49 --------------- 10 files changed, 253 deletions(-) delete mode 100644 .buildkite/test_areas/weight_loading.yaml delete mode 100644 tests/weight_loading/models-amd.txt delete mode 100644 tests/weight_loading/models-large-amd.txt delete mode 100644 tests/weight_loading/models-large.txt delete mode 100644 tests/weight_loading/models.txt delete mode 100755 tests/weight_loading/run_model_weight_loading_test.sh delete mode 100644 tests/weight_loading/test_weight_loading.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 2f5a84aaf1..ba8f382755 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3335,36 +3335,6 @@ steps: commands: - pytest -v -s v1/e2e/test_hybrid_chunked_prefill.py -#------------------------------------------------------ mi300 · weight_loading -------------------------------------------------------# - -- label: ":amd: (MI300) Weight Loading Multi-GPU" # TBD - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - num_gpus: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/weight_loading - optional: true - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt - -- label: ":amd: (MI300) Weight Loading Multi-GPU (Large Models)" # TBD - timeout_in_minutes: 35 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - num_gpus: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/weight_loading - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt - ######################################################################################################################################### # # # MI355 (gfx950) tests # @@ -4719,36 +4689,6 @@ steps: - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi -#------------------------------------------------------ mi355 · weight_loading -------------------------------------------------------# - -- label: ":amd: (MI355) Weight Loading Multi-GPU" # TBD - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_2 - num_gpus: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/weight_loading - optional: true - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt - -- label: ":amd: (MI355) Weight Loading Multi-GPU (Large Models)" # TBD - timeout_in_minutes: 30 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_2 - working_dir: "/vllm-workspace/tests" - num_gpus: 2 - optional: true - source_file_dependencies: - - vllm/ - - tests/weight_loading - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt - #----------------------------------------------------------- mi355 · misc ------------------------------------------------------------# - label: ":amd: (MI355) Regression" # TBD diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml deleted file mode 100644 index b8afa261e5..0000000000 --- a/.buildkite/test_areas/weight_loading.yaml +++ /dev/null @@ -1,38 +0,0 @@ -group: Weight Loading -depends_on: - - image-build -steps: -- label: ":nvidia: (L4) Weight Loading Multi-GPU" - key: weight-loading-multiple-gpu - timeout_in_minutes: 50 - working_dir: "/vllm-workspace/tests" - device: l4 - num_devices: 2 - optional: true - source_file_dependencies: - - vllm/ - - "!vllm/distributed/kv_transfer/" - - tests/weight_loading - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt - mirror: - amd: - label: ":amd: (MI355) Weight Loading Multi-GPU" - dind: false - device: mi355_2 - timeout_in_minutes: 70 - depends_on: - - image-build-amd - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt - -# - label: Weight Loading Multiple GPU - Large Models # optional -# working_dir: "/vllm-workspace/tests" -# num_devices: 2 -# device: a100 -# optional: true -# source_file_dependencies: -# - vllm/ -# - tests/weight_loading -# commands: -# - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aa5b2a9fc9..f64cf5f561 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -110,7 +110,6 @@ /tests/entrypoints/llm/test_struct_output_generate.py @mgoin @aarnphm @arpera /tests/v1/structured_output @mgoin @aarnphm @arpera /tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium -/tests/weight_loading @mgoin @youkaichao @yewentao256 /tests/lora @jeejeelee /tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91 /tests/v1/kv_connector/nixl_integration @NickLucche diff --git a/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py b/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py index a9b35a4ea6..8e4563ab80 100644 --- a/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py +++ b/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py @@ -17,7 +17,6 @@ from vllm.platforms import current_platform @pytest.mark.parametrize( "model_path", [ - # Listed in tests/weight_loading/models.txt "nm-testing/tinyllama-oneshot-w4a16-group128-v2", ], ) diff --git a/tests/weight_loading/models-amd.txt b/tests/weight_loading/models-amd.txt deleted file mode 100644 index a6a85736cb..0000000000 --- a/tests/weight_loading/models-amd.txt +++ /dev/null @@ -1,18 +0,0 @@ -# quantization, model, revision[, minimum capability[, safetensors load strategy]] -fp8, amd/Meta-Llama-3.1-8B-Instruct-FP8-KV, main -None, amd/Llama-3.2-1B-Instruct-FP8-KV, main -fp8, amd/Mixtral-8x7B-Instruct-v0.1-FP8-KV, main, 80, prefetch -gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main -gptq, TheBloke/Llama-2-7B-GPTQ, main -gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True -compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w4a16-group128-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main -compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main -compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main -compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main -awq, casperhansen/mixtral-instruct-awq, main -fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main diff --git a/tests/weight_loading/models-large-amd.txt b/tests/weight_loading/models-large-amd.txt deleted file mode 100644 index 41e8ee5200..0000000000 --- a/tests/weight_loading/models-large-amd.txt +++ /dev/null @@ -1,4 +0,0 @@ -# quantization, model, revision[, minimum capability[, safetensors load strategy]] -fp8, amd/Meta-Llama-3.1-70B-Instruct-FP8-KV, main, 80, prefetch -None, microsoft/phi-4, main, 80, prefetch -fp8, amd/Mixtral-8x22B-Instruct-v0.1-FP8-KV, main, 80, prefetch diff --git a/tests/weight_loading/models-large.txt b/tests/weight_loading/models-large.txt deleted file mode 100644 index 8b3913cc2d..0000000000 --- a/tests/weight_loading/models-large.txt +++ /dev/null @@ -1,6 +0,0 @@ -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main -gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main -awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main -compressed-tensors, RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16, main diff --git a/tests/weight_loading/models.txt b/tests/weight_loading/models.txt deleted file mode 100644 index 87b9650870..0000000000 --- a/tests/weight_loading/models.txt +++ /dev/null @@ -1,18 +0,0 @@ -gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main -gptq, TheBloke/Llama-2-7B-GPTQ, main -gptq, LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit, main -gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True -gptq, TechxGenus/gemma-2b-it-GPTQ, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w4a16-group128-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main -compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main -compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main -compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main -#compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main -awq, casperhansen/mixtral-instruct-awq, main -awq_marlin, casperhansen/mixtral-instruct-awq, main -fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main diff --git a/tests/weight_loading/run_model_weight_loading_test.sh b/tests/weight_loading/run_model_weight_loading_test.sh deleted file mode 100755 index 598f1620c7..0000000000 --- a/tests/weight_loading/run_model_weight_loading_test.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -SUCCESS=0 - -while getopts "c:" OPT; do - case ${OPT} in - c ) - CONFIG="$OPTARG" - ;; - \? ) - usage - exit 1 - ;; - esac -done - - -IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG" - -for MODEL_CONFIG in "${MODEL_CONFIGS[@]}" -do - if [[ $MODEL_CONFIG == \#* ]]; then - echo "=== SKIPPING MODEL: $MODEL_CONFIG ===" - continue - fi - - LOCAL_SUCCESS=0 - IFS=', ' read -r -a array <<< "$MODEL_CONFIG" - - echo "=== RUNNING MODEL: $MODEL_CONFIG ===" - - unset WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY - export QUANTIZATION=${array[0]} - export MODEL_NAME=${array[1]} - export REVISION=${array[2]} - # If array length is larger than 3, then MIN_CAPABILITY is provided - if [ ${#array[@]} -gt 3 ]; then - export MIN_CAPABILITY=${array[3]} - fi - if [ ${#array[@]} -gt 4 ]; then - export WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY=${array[4]} - fi - pytest -s weight_loading/test_weight_loading.py || LOCAL_SUCCESS=$? - - if [[ $LOCAL_SUCCESS == 0 ]]; then - echo "=== PASSED MODEL: ${MODEL_CONFIG} ===" - else - echo "=== FAILED MODEL: ${MODEL_CONFIG} ===" - fi - - SUCCESS=$((SUCCESS + LOCAL_SUCCESS)) - -done - -if [ "${SUCCESS}" -eq "0" ]; then - exit 0 -else - exit 1 -fi diff --git a/tests/weight_loading/test_weight_loading.py b/tests/weight_loading/test_weight_loading.py deleted file mode 100644 index c32ce71aca..0000000000 --- a/tests/weight_loading/test_weight_loading.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import os - -import pytest -import torch - -from vllm.platforms import current_platform -from vllm.utils.mem_constants import GiB_bytes - -MAX_MODEL_LEN = 1024 -MODEL_NAME = os.environ.get( - "MODEL_NAME", "robertgshaw2/zephyr-7b-beta-channelwise-gptq" -) -REVISION = os.environ.get("REVISION", "main") -QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin") -MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "80") -SAFETENSORS_LOAD_STRATEGY = os.environ.get( - "WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY" -) - - -@pytest.mark.skipif( - MODEL_NAME == "casperhansen/deepseek-coder-v2-instruct-awq", reason="OOM in the CI" -) -@pytest.mark.skipif( - not current_platform.has_device_capability(int(MIN_CAPABILITY)), - reason="Current system does not have minimum capability.", -) -def test_weight_loading(vllm_runner): - """Test parameter weight loading with tp>1.""" - # MoE models need fp16. - NEEDS_FP16 = QUANTIZATION == "gptq" - with vllm_runner( - model_name=MODEL_NAME, - revision=REVISION, - dtype=torch.half if NEEDS_FP16 else "auto", - quantization=None if QUANTIZATION == "None" else QUANTIZATION, - max_model_len=MAX_MODEL_LEN, - tensor_parallel_size=2, - safetensors_load_strategy=SAFETENSORS_LOAD_STRATEGY, - # Generating 20 tokens needs a tiny KV cache, while sizing and - # allocating a full-size one dominates this test on large devices. - kv_cache_memory_bytes=2 * GiB_bytes, - ) as model: - output = model.generate_greedy("Hello world!", max_tokens=20) - print(output) - assert output From 2b02c6c29b72147de18905ba4e7dc8f284655b40 Mon Sep 17 00:00:00 2001 From: Doug Brown Date: Thu, 17 Sep 2026 16:09:36 -0400 Subject: [PATCH 055/143] [Model] Add Cohere2MoE Eagle3 auxiliary hidden states (#49819) Signed-off-by: Doug Brown --- tests/v1/spec_decode/test_dflash_causality.py | 12 ++++++++++ vllm/model_executor/models/cohere2_moe.py | 23 ++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/tests/v1/spec_decode/test_dflash_causality.py b/tests/v1/spec_decode/test_dflash_causality.py index 0f1de86cf9..dc13e55700 100644 --- a/tests/v1/spec_decode/test_dflash_causality.py +++ b/tests/v1/spec_decode/test_dflash_causality.py @@ -103,6 +103,18 @@ def test_dflash_fc_uses_aux_layer_count(): assert _get_dflash_fc_input_size(vllm_config) == 3 * 4096 +def test_dflash_target_layers_map_to_post_layer_boundaries(): + vllm_config = _vllm_config( + dflash_config={"target_layer_ids": [0, 16, 31]}, + ) + + assert get_eagle3_aux_layers_from_config(vllm_config.speculative_config) == ( + 1, + 17, + 32, + ) + + @pytest.mark.parametrize("config_name", ["dflash_config", "eagle_config"]) def test_eagle_aux_layers_preserves_legacy_layer_ids(config_name): layer_ids = [1, 17, 32] diff --git a/vllm/model_executor/models/cohere2_moe.py b/vllm/model_executor/models/cohere2_moe.py index d657b78a5d..98e649dec6 100644 --- a/vllm/model_executor/models/cohere2_moe.py +++ b/vllm/model_executor/models/cohere2_moe.py @@ -35,7 +35,7 @@ from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from .commandr import LayerNorm -from .interfaces import SupportsPP, SupportsQuant +from .interfaces import EagleModelMixin, SupportsEagle3, SupportsPP, SupportsQuant from .utils import ( AutoWeightsLoader, WeightsMapper, @@ -385,7 +385,7 @@ class Cohere2MoeDecoderLayer(nn.Module): @support_torch_compile -class Cohere2MoeModel(nn.Module): +class Cohere2MoeModel(nn.Module, EagleModelMixin): """Transformer decoder for Cohere2Moe.""" def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): @@ -439,7 +439,7 @@ class Cohere2MoeModel(nn.Module): positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: + ) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]: if get_pp_group().is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds @@ -450,17 +450,28 @@ class Cohere2MoeModel(nn.Module): assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] - for layer in islice(self.layers, self.start_layer, self.end_layer): + + aux_hidden_states = [] + if self.start_layer in self.aux_hidden_state_layers: + aux_hidden_states.append(hidden_states) + for layer_idx, layer in enumerate( + islice(self.layers, self.start_layer, self.end_layer), + start=self.start_layer, + ): hidden_states, residual = layer(positions, hidden_states, residual) + if layer_idx + 1 in self.aux_hidden_state_layers: + aux_hidden_states.append(hidden_states) if not get_pp_group().is_last_rank: return IntermediateTensors( {"hidden_states": hidden_states, "residual": residual} ) hidden_states, _ = self.norm(hidden_states, residual) + if aux_hidden_states: + return hidden_states, aux_hidden_states return hidden_states -class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant): +class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant, SupportsEagle3): is_text_generation_model = True hf_to_vllm_mapper = WeightsMapper( @@ -520,7 +531,7 @@ class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant): positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: + ) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]: return self.model(input_ids, positions, intermediate_tensors, inputs_embeds) def compute_logits( From acc2ed2a5f1fbe79575a220fc743a80c2d068ce8 Mon Sep 17 00:00:00 2001 From: rasmith Date: Thu, 17 Sep 2026 15:20:35 -0500 Subject: [PATCH 056/143] [ROCm][CI][The Rock 10] Fix (MI355) Quantized Models failure on The Rock 10 with Triton 3.8.x (#54849) Signed-off-by: Randall Smith Co-authored-by: Shanshan Shen <467638484@qq.com> --- .../kernels/linear/mxfp8/rocm_native.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py index 29739afae3..1d9789cced 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py +++ b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py @@ -119,6 +119,18 @@ def _mxfp8_dot_scaled_linear( return out +# Triton 3.8 enables TRITON_HIP_USE_ASYNC_COPY by default on gfx950; its extra LDS +# buffer leaves no room for num_stages=3 at BLOCK_K=256. +# TODO(rasmith)(Rohan138): Remove the 3.8 check once +# https://github.com/vllm-project/vllm/pull/50605 merges. +_BK256_STAGES = 3 +if triton.__version__.startswith("3.8") and current_platform.is_rocm(): + from vllm.platforms.rocm import on_gfx950 + + if on_gfx950(): + _BK256_STAGES = 2 + + def _select_cfg(M, N, K): """(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) — graph-tuned on gfx950. @@ -166,7 +178,7 @@ def _select_cfg(M, N, K): # 3.6; on triton 3.7 its large BLOCK_M register/LDS footprint spills or hits # "out of resources", so use 128x128x256 -- within the known-good footprint.) if M >= 4096 and K >= 1024 and K % 256 == 0 and occ >= 256: - return 128, 128, 256, 8, 3 + return 128, 128, 256, 8, _BK256_STAGES return 128, 128, 128, 8, 3 # large-K (K >= 2048). BLOCK_K is K-divisibility-guarded (the K-loop is unmasked): # served large-K is 2048/6144 (%256==0), but fall back to 128 (always divides, since @@ -186,7 +198,7 @@ def _select_cfg(M, N, K): # Covers the qkv-class local N=1536 (TP=8 qkv / TP=4 shared_gate_up) and the deep-K # / very-large-M shapes. if K % 256 == 0 and (1280 < N <= 1536 or (occ >= 128 and (K >= 4096 or M >= 4096))): - return 128, 128, 256, 8, 3 + return 128, 128, 256, 8, _BK256_STAGES # small local-N (e.g. TP=8 shared_gate_up N=768): a 64-wide BLOCK_N doubles the # N-tile count -> better CU fill than 128x128 at this mid-large M (~1.4x there). if N <= 1024 and K % 256 == 0: From 9612f77077e09acbae9cc1d1b2ddf14a23e91597 Mon Sep 17 00:00:00 2001 From: Maroon Ayoub Date: Thu, 17 Sep 2026 23:23:15 +0300 Subject: [PATCH 057/143] [Bugfix] Release stale FlashMLA workspace views after growth (#56902) Signed-off-by: Maroon Ayoub Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- .../attention/backends/mla/flashmla_sparse.py | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 265fb779c3..b3332eb862 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -720,7 +720,9 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): ) self.pcp_dcp_kv_gather = False - self.gathered_kv_workspace: torch.Tensor | None = None + self.workspace_specs: list[tuple[tuple[int, ...], torch.dtype]] = [ + (q_concat_shape, torch.bfloat16) + ] if kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS: # Reserve workspace during initialization assert vllm_config is not None and vllm_config.model_config is not None @@ -738,20 +740,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): # shards into a workspace of the full prefill size. shard_rows //= parallel_config.decode_context_parallel_size self.prefill_workspace_shape = (shard_rows, head_size) - specs = [ - (q_concat_shape, torch.bfloat16), - (self.prefill_workspace_shape, torch.bfloat16), - ] + self.workspace_specs.append((self.prefill_workspace_shape, torch.bfloat16)) if self.pcp_dcp_kv_gather: - specs.append(((prefill_workspace_size, head_size), torch.bfloat16)) - buffers = current_workspace_manager().get_simultaneous(*specs) - self.q_concat_buffer, self.prefill_bf16_workspace = buffers[:2] - if self.pcp_dcp_kv_gather: - self.gathered_kv_workspace = buffers[2] - else: - (self.q_concat_buffer,) = current_workspace_manager().get_simultaneous( - (q_concat_shape, torch.bfloat16), - ) + self.workspace_specs.append( + ((prefill_workspace_size, head_size), torch.bfloat16) + ) + # Reserve capacity without retaining views that prevent old storage + # from being released when another layer grows the shared workspace. + current_workspace_manager().get_simultaneous(*self.workspace_specs) def _forward_bf16_kv( self, @@ -849,8 +845,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): """All-gather this rank's upconverted KV shard so the chunk's rows attend the whole context, and map their top-k onto the rank-major result.""" shard_rows = int(chunk.chunk_tot_seqlen) - assert self.gathered_kv_workspace is not None - gathered_kv = self.gathered_kv_workspace[: self.dcp_world_size * shard_rows] + _, _, gathered_kv_workspace = current_workspace_manager().get_simultaneous( + *self.workspace_specs + ) + gathered_kv = gathered_kv_workspace[: self.dcp_world_size * shard_rows] dist.all_gather_into_tensor( gathered_kv, shard, group=get_dcp_group().device_group ) @@ -902,10 +900,13 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): assert fp8_metadata.prefill is not None first_chunk = fp8_metadata.prefill.chunks[0] assert isinstance(index_group, HiSparseMLAIndexGroup) + _, prefill_bf16_workspace, *_ = ( + current_workspace_manager().get_simultaneous(*self.workspace_specs) + ) prefill_ready = index_group.gather_fp8_prefill( self.index_group_index, kv_c_and_k_pe_cache, - self.prefill_bf16_workspace[: first_chunk.chunk_tot_seqlen], + prefill_bf16_workspace[: first_chunk.chunk_tot_seqlen], first_chunk.block_table, first_chunk.workspace_starts, len(first_chunk.block_table), @@ -1022,8 +1023,11 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): ) assert fp8_metadata.prefill is not None + _, prefill_bf16_workspace, *_ = ( + current_workspace_manager().get_simultaneous(*self.workspace_specs) + ) for chunk_index, chunk in enumerate(fp8_metadata.prefill.chunks): - chunk_workspace = self.prefill_bf16_workspace[: chunk.chunk_tot_seqlen] + chunk_workspace = prefill_bf16_workspace[: chunk.chunk_tot_seqlen] if uses_host_cache and chunk_index > 0: assert isinstance(index_group, HiSparseMLAIndexGroup) prefill_ready = index_group.gather_fp8_prefill( @@ -1302,7 +1306,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): actual_num_heads = self.num_heads if isinstance(q, tuple): ql_nope, q_pe = q - q = self.q_concat_buffer[: ql_nope.shape[0]] + q_concat_buffer, *_ = current_workspace_manager().get_simultaneous( + *self.workspace_specs + ) + q = q_concat_buffer[: ql_nope.shape[0]] ops.concat_mla_q(ql_nope, q_pe, q) else: actual_num_heads = q.shape[1] From 668d6c3a778b9ac525f78d39b0c413c8604235d1 Mon Sep 17 00:00:00 2001 From: Thang Nguyen <69278249+Thangnguyenvn98@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:11:40 -0700 Subject: [PATCH 058/143] [CI] Shard multimodal Processor 1->4 (#56316) Signed-off-by: Thang Nguyen Co-authored-by: Thang Nguyen Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Kimi Code --- .buildkite/test_areas/models_multimodal.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index e8c5ddd2cb..a324da687f 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -109,22 +109,24 @@ steps: depends_on: - image-build-amd -- label: ":nvidia: (H200 MIG 18GB) Multimodal Processor" +- label: ":nvidia: (H200 MIG 18GB) Multimodal Processor Shard %N" key: multi-modal-processor timeout_in_minutes: 98 device: h200_18gb + parallelism: 4 source_file_dependencies: - vllm/ - "!vllm/distributed/kv_transfer/" - tests/models/multimodal - tests/models/registry.py commands: - - pytest -v -s models/multimodal/processing/test_tensor_schema.py + - pytest -v -s models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: - label: ":amd: (MI355) Multimodal Processor" + label: ":amd: (MI355) Multimodal Processor Shard %N" dind: false device: mi355_1 + parallelism: 4 timeout_in_minutes: 115 depends_on: - image-build-amd From db7a24c230a4db6ae3a568ed01b01f3f7172069d Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Thu, 17 Sep 2026 17:38:59 -0400 Subject: [PATCH 059/143] [Perf] Add fused DFlash2 grouped convolution (#55960) Signed-off-by: mgoin --- .buildkite/test_areas/spec_decode.yaml | 4 + .../acceptance_rates/dflash/test_dflash.py | 21 +++ tests/v1/spec_decode/test_dflash2.py | 50 +++++++ vllm/model_executor/models/qwen3_dflash2.py | 124 ++++++++++++++++++ 4 files changed, 199 insertions(+) diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index c06642ea1a..fdb12efe11 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -283,6 +283,8 @@ steps: - vllm/v1/spec_decode/ - vllm/v1/worker/gpu/spec_decode/ - vllm/model_executor/models/qwen3_dflash.py + - vllm/model_executor/models/qwen3_dflash2.py + - vllm/model_executor/models/qwen3_5.py - vllm/model_executor/models/laguna_dflash.py - tests/v1/e2e/spec_decode/ commands: @@ -293,6 +295,8 @@ steps: dind: false device: mi300_1 timeout_in_minutes: 120 + commands: + - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "not dflash2" depends_on: - image-build-amd source_file_dependencies: diff --git a/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py b/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py index 2b1800f782..48dcde2bbc 100644 --- a/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py +++ b/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py @@ -26,6 +26,8 @@ class DFlashCorrectnessConfig: use_chat_completions: bool = False enforce_eager: bool = False disable_flashinfer_sampler: bool = False + language_model_only: bool = False + chat_template_kwargs: dict[str, object] | None = None QWEN3_DFLASH = DFlashCorrectnessConfig( @@ -35,6 +37,18 @@ QWEN3_DFLASH = DFlashCorrectnessConfig( expected_acceptance_len=3.5, ) +QWEN3_8_DFLASH2_NVFP4 = DFlashCorrectnessConfig( + model="nvidia/Qwen3.8-27B-NVFP4", + draft_model="z-lab/Qwen3.8-27B-DFlash2", + expected_accuracy=0.90, + expected_acceptance_len=6.14 * 0.95, + num_speculative_tokens=7, + max_num_seqs=32, + use_chat_completions=True, + language_model_only=True, + chat_template_kwargs={"enable_thinking": False}, +) + LAGUNA_DFLASH_NVFP4 = DFlashCorrectnessConfig( model="poolside/Laguna-XS-2.1-NVFP4", draft_model="poolside/Laguna-XS-2.1-DFlash-NVFP4", @@ -104,6 +118,11 @@ def test_dflash_reference_acceptance_lengths( True, id="laguna-nvfp4-mrv2", ), + pytest.param( + QWEN3_8_DFLASH2_NVFP4, + True, + id="qwen3.8-dflash2-nvfp4-mrv2", + ), ], ) def test_dflash_correctness( @@ -134,12 +153,14 @@ def test_dflash_correctness( disable_log_stats=False, enable_chunked_prefill=None, compilation_config=CompilationConfig(), + language_model_only=config.language_model_only, ) as spec_runner: spec_llm = spec_runner.llm results = evaluate_gsm8k_offline( spec_llm, num_questions=config.num_questions, use_chat_completions=config.use_chat_completions, + chat_template_kwargs=config.chat_template_kwargs, ) accuracy = results["accuracy"] acceptance_len = compute_acceptance_len(spec_llm.get_metrics()) diff --git a/tests/v1/spec_decode/test_dflash2.py b/tests/v1/spec_decode/test_dflash2.py index e7ff2b6d5d..72379fa7d7 100644 --- a/tests/v1/spec_decode/test_dflash2.py +++ b/tests/v1/spec_decode/test_dflash2.py @@ -7,6 +7,7 @@ import pytest import torch from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges +from vllm.platforms import current_platform from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator @@ -35,6 +36,55 @@ def test_grouped_conv_matches_reference(block_size: int): torch.testing.assert_close(actual, expected.flatten(0, 1).flatten(-2)) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize( + "batch,num_groups,group_size,block_size,taps", + [ + (7, 13, 11, 5, 1), + (7, 13, 11, 5, 3), + (7, 13, 11, 8, 2), + (16, 64, 16, 8, 2), + (16, 160, 16, 8, 2), + ], +) +def test_grouped_conv_triton_matches_reference( + dtype: torch.dtype, + batch: int, + num_groups: int, + group_size: int, + block_size: int, + taps: int, +): + torch.manual_seed(0) + rows = batch * block_size + hidden = torch.randn(rows, num_groups * group_size, device="cuda", dtype=dtype) + base = torch.randn(taps, num_groups * group_size, device="cuda", dtype=dtype) + projected = torch.randn(rows, 2, taps, num_groups, device="cuda", dtype=dtype) + delta = projected[:, 1] + + actual = _grouped_conv( + hidden, delta, base, block_size, num_groups, group_size, taps + ) + + hidden_blocks = hidden.float().view(batch, block_size, num_groups, group_size) + expected = torch.zeros_like(hidden_blocks) + base_blocks = base.float().view(taps, num_groups, group_size) + delta_blocks = delta.float().view(batch, block_size, taps, num_groups) + for position in range(block_size): + for tap in range(min(taps, position + 1)): + expected[:, position] += ( + base_blocks[tap] + delta_blocks[:, position, tap, :, None] + ) * hidden_blocks[:, position - tap] + + torch.testing.assert_close( + actual, + expected.flatten(0, 1).flatten(-2).to(dtype), + rtol=1e-2 if dtype is torch.bfloat16 else 1e-5, + atol=1e-2 if dtype is torch.bfloat16 else 1e-5, + ) + + def test_selector_edges_match_sequential_reference(): torch.manual_seed(1) batch, steps, top_k, rank = 2, 4, 3, 5 diff --git a/vllm/model_executor/models/qwen3_dflash2.py b/vllm/model_executor/models/qwen3_dflash2.py index cb027293f9..2043307d02 100644 --- a/vllm/model_executor/models/qwen3_dflash2.py +++ b/vllm/model_executor/models/qwen3_dflash2.py @@ -11,6 +11,8 @@ from vllm.config import CacheConfig, VllmConfig from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op from .qwen3_dflash import ( DFlashQwen3DecoderLayer, @@ -20,6 +22,125 @@ from .qwen3_dflash import ( from .utils import maybe_prefix +@triton.jit +def _dflash2_grouped_conv_kernel( + x_ptr, + delta_ptr, + base_ptr, + output_ptr, + x_stride_row, + delta_stride_row, + delta_stride_tap, + base_stride_tap, + output_stride_row, + NUM_CHANNELS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + GROUP_SIZE: tl.constexpr, + TAPS: tl.constexpr, + ELEMENT_BLOCK: tl.constexpr, +) -> None: + row = tl.program_id(0) // triton.cdiv(NUM_CHANNELS, ELEMENT_BLOCK) + col_block = tl.program_id(0) % triton.cdiv(NUM_CHANNELS, ELEMENT_BLOCK) + channels = col_block * ELEMENT_BLOCK + tl.arange(0, ELEMENT_BLOCK) + mask = channels < NUM_CHANNELS + groups = channels // GROUP_SIZE + position = row % BLOCK_SIZE + + delta_row = delta_ptr + row * delta_stride_row + x_row = x_ptr + row * x_stride_row + accumulator = ( + tl.load(base_ptr + channels, mask=mask, other=0.0).to(tl.float32) + + tl.load(delta_row + groups, mask=mask, other=0.0).to(tl.float32) + ) * tl.load(x_row + channels, mask=mask, other=0.0).to(tl.float32) + + for tap in tl.static_range(1, TAPS): + tap_mask = mask & (position >= tap) + coefficient = tl.load( + base_ptr + tap * base_stride_tap + channels, + mask=tap_mask, + other=0.0, + ).to(tl.float32) + tl.load( + delta_row + tap * delta_stride_tap + groups, + mask=tap_mask, + other=0.0, + ).to(tl.float32) + x = tl.load( + x_ptr + (row - tap) * x_stride_row + channels, + mask=tap_mask, + other=0.0, + ).to(tl.float32) + accumulator += coefficient * x + + tl.store( + output_ptr + row * output_stride_row + channels, + accumulator, + mask=mask, + ) + + +def dflash2_grouped_conv_impl( + x: torch.Tensor, + delta: torch.Tensor, + base: torch.Tensor, + block_size: int, + group_size: int, +) -> torch.Tensor: + """DFlash2 grouped convolution adapted from LightSeek TokenSpeed PR #1399.""" + num_rows, num_channels = x.shape + output = torch.empty_like(x) + if num_rows == 0: + return output + + element_block = 1024 if num_rows >= 128 and num_channels % 1024 == 0 else 512 + grid = (num_rows * triton.cdiv(num_channels, element_block),) + _dflash2_grouped_conv_kernel[grid]( + x, + delta, + base, + output, + x.stride(0), + delta.stride(0), + delta.stride(1), + base.stride(0), + output.stride(0), + NUM_CHANNELS=num_channels, + BLOCK_SIZE=block_size, + GROUP_SIZE=group_size, + TAPS=base.shape[0], + ELEMENT_BLOCK=element_block, + num_warps=4, + ) + return output + + +def dflash2_grouped_conv_fake( + x: torch.Tensor, + delta: torch.Tensor, + base: torch.Tensor, + block_size: int, + group_size: int, +) -> torch.Tensor: + return torch.empty_like(x) + + +direct_register_custom_op( + op_name="dflash2_grouped_conv", + op_func=dflash2_grouped_conv_impl, + fake_impl=dflash2_grouped_conv_fake, + tags=(torch.Tag.needs_fixed_stride_order,), +) + + +def dflash2_grouped_conv( + x: torch.Tensor, + delta: torch.Tensor, + base: torch.Tensor, + block_size: int, + group_size: int, +) -> torch.Tensor: + return torch.ops.vllm.dflash2_grouped_conv(x, delta, base, block_size, group_size) + + def _grouped_conv( hidden_states: torch.Tensor, delta: torch.Tensor, @@ -29,6 +150,9 @@ def _grouped_conv( group_size: int, taps: int, ) -> torch.Tensor: + if hidden_states.is_cuda: + return dflash2_grouped_conv(hidden_states, delta, base, block_size, group_size) + blocks = hidden_states.unflatten(-1, (num_groups, group_size)) coefficients = base.view(1, taps, num_groups, group_size) + delta.unsqueeze(-1) output = coefficients[:, 0] * blocks From ac78445bc9ee3bb615b8b21db22fb7dac6698ece Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Thu, 17 Sep 2026 17:47:30 -0400 Subject: [PATCH 060/143] [MoE] Encapsulate TRT-LLM BF16 weight layout handling (#57405) Signed-off-by: mgoin --- tests/kernels/moe/test_flashinfer.py | 30 ++++-- .../moe/test_moe_weight_loading_padded.py | 25 ----- tests/kernels/moe/test_trtllm_bf16_moe.py | 49 +++++++++ .../fused_moe/experts/trtllm_bf16_moe.py | 17 ++- .../fused_moe/experts/trtllm_lora_moe.py | 7 +- .../layers/fused_moe/oracle/unquantized.py | 27 ++++- .../fused_moe/unquantized_fused_moe_method.py | 100 +++++------------- 7 files changed, 138 insertions(+), 117 deletions(-) diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index 53d760c339..d17416de99 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( FlashInferExperts, ) +from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( + view_as_block_major_k, +) from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import ( TrtLlmFp8ExpertsModular, TrtLlmFp8ExpertsMonolithic, @@ -570,6 +573,10 @@ def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed( padded = w2_shape[-1] w13_ptr = layer.w13_weight.data_ptr() w2_ptr = layer.w2_weight.data_ptr() + load_numel = ( + layer.w13_weight.weight_loader_numel, + layer.w2_weight.weight_loader_numel, + ) for _ in range(2): reloaded_w13 = torch.randn_like(layer.w13_weight) @@ -602,19 +609,24 @@ def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed( assert layer.w2_weight.shape == w2_shape assert layer.w13_weight.data_ptr() == w13_ptr assert layer.w2_weight.data_ptr() == w2_ptr - kernel_w13, kernel_w2 = method._kernel_weights(layer) + assert ( + layer.w13_weight.weight_loader_numel, + layer.w2_weight.weight_loader_numel, + ) == load_numel + kernel_w13 = view_as_block_major_k(layer.w13_weight) + kernel_w2 = view_as_block_major_k(layer.w2_weight) assert torch.equal(kernel_w13, expected_w13) assert torch.equal(kernel_w2, expected_w2) -def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_index): +def _check_flashinfer_ipc_weights(entries, expected, mode, device_index): from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader from vllm.model_executor.utils import weights_already_processed torch.accelerator.set_device_index(device_index) with pytest.MonkeyPatch.context() as monkeypatch: method, layer = _make_unquantized_flashinfer_test_layer( - monkeypatch, 192, is_gated, device="meta" + monkeypatch, 192, is_gated=True, device="meta" ) loader = object.__new__(IpcModelLoader) loader.mode = mode @@ -625,7 +637,10 @@ def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_inde with weights_already_processed(): method.process_weights_after_loading(layer) assert method.moe_kernel is not None - actual = method._kernel_weights(layer) + actual = ( + view_as_block_major_k(layer.w13_weight), + view_as_block_major_k(layer.w2_weight), + ) for weight, reference, pointer in zip(actual, expected, pointers): assert weight.data_ptr() == pointer assert torch.equal(weight.cpu(), reference) @@ -636,11 +651,10 @@ def _check_flashinfer_ipc_weights(entries, expected, is_gated, mode, device_inde torch.accelerator.synchronize() -@pytest.mark.parametrize("is_gated", [True, False]) @pytest.mark.parametrize("cache_ndim", [3, 4]) @pytest.mark.parametrize("mode", ["copy", "zero_copy"]) def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state( - monkeypatch, is_gated, cache_ndim, mode + monkeypatch, cache_ndim, mode ): """A spawned IPC consumer restores views without transient method state.""" from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( @@ -649,13 +663,12 @@ def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state( from vllm.model_executor.model_loader.weight_cache.protocol import TensorEntry method, layer = _make_unquantized_flashinfer_test_layer( - monkeypatch, 192, is_gated, device="meta" + monkeypatch, 192, is_gated=True, device="meta" ) packed = convert_moe_weights_to_flashinfer_trtllm_block_layout( {}, torch.randn_like(layer.w13_weight, device="cuda"), torch.randn_like(layer.w2_weight, device="cuda"), - is_gated_act_gemm=is_gated, ) expected = [weight.cpu() for weight in packed] entries = {} @@ -667,7 +680,6 @@ def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state( args=( entries, expected, - is_gated, mode, torch.accelerator.current_device_index(), ), diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py index 423f181132..ef31de06d1 100644 --- a/tests/kernels/moe/test_moe_weight_loading_padded.py +++ b/tests/kernels/moe/test_moe_weight_loading_padded.py @@ -12,7 +12,6 @@ correctly handles this mismatch. import pytest import torch -from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( UnquantizedMoeBackend, ) @@ -465,30 +464,6 @@ class TestUnquantizedTrtLlmPrePadding: assert hidden == 64 assert intermediate == expected - @pytest.mark.parametrize("is_act_and_mul", [True, False]) - def test_create_weights_zeroes_preallocated_padding(self, is_act_and_mul: bool): - method = self._make_method(1344) - if not is_act_and_mul: - method.moe.activation = MoEActivation.RELU2_NO_MUL - method.moe.intermediate_size_per_partition = 1408 - layer = torch.nn.Module() - - method.create_weights( - layer=layer, - num_experts=2, - hidden_size=64, - intermediate_size_per_partition=1408, - params_dtype=torch.bfloat16, - ) - - up_mult = 2 if is_act_and_mul else 1 - assert layer.w13_weight.shape == (2, up_mult * 1408, 64) - assert layer.w2_weight.shape == (2, 64, 1408) - assert torch.count_nonzero(layer.w13_weight[:, 1344:1408]) == 0 - if is_act_and_mul: - assert torch.count_nonzero(layer.w13_weight[:, 2752:]) == 0 - assert torch.count_nonzero(layer.w2_weight[:, :, 1344:]) == 0 - class TestLoadWeightsExpertBias: """Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize diff --git a/tests/kernels/moe/test_trtllm_bf16_moe.py b/tests/kernels/moe/test_trtllm_bf16_moe.py index f7466c63c8..5e569e153a 100644 --- a/tests/kernels/moe/test_trtllm_bf16_moe.py +++ b/tests/kernels/moe/test_trtllm_bf16_moe.py @@ -12,6 +12,7 @@ import pytest import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from tests.kernels.moe.utils import make_dummy_moe_config from tests.kernels.utils import torch_moe from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -27,6 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( TrtLlmBf16ExpertsModular, ) +from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import ( + TrtLlmBf16LoRAExperts, +) from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( UnquantizedMoeBackend, convert_to_unquantized_kernel_format, @@ -150,3 +154,48 @@ def test_trtllm_bf16_moe_modular_no_graph( atol=1e-1, rtol=2e-1, ) + + +@pytest.mark.parametrize("has_lora_delta", [False, True]) +@torch.inference_mode() +def test_trtllm_bf16_lora_accepts_checkpoint_shaped_weights(has_lora_delta): + """LoRA dispatch accepts both 3D parameters and legacy 4D packed views.""" + config = make_dummy_moe_config( + num_experts=128, hidden_dim=256, intermediate_size=128 + ) + experts = TrtLlmBf16LoRAExperts(config, FUSED_MOE_UNQUANTIZED_CONFIG) + x = torch.randn(2, 256, device="cuda", dtype=torch.bfloat16) / 10 + w1, w2 = convert_to_unquantized_kernel_format( + UnquantizedMoeBackend.FLASHINFER_TRTLLM, + config, + torch.randn(128, 256, 256, device="cuda", dtype=torch.bfloat16) / 10, + torch.randn(128, 256, 128, device="cuda", dtype=torch.bfloat16) / 10, + ) + assert w1.ndim == w2.ndim == 3 + topk = ( + torch.tensor([[0], [1]], device="cuda", dtype=torch.int32), + torch.ones(2, 1, device="cuda", dtype=torch.float32), + ) + delta = torch.randn(2, 1, 256, device="cuda", dtype=torch.bfloat16) + + def invoke(w1, w2): + result = experts.invoke_routed_moe( + hidden_states=x, + w1=w1, + w2=w2, + topk_ids_and_weights=topk, + gemm1_lora_delta=delta if has_lora_delta else None, + global_num_experts=128, + a1q_scale=None, + output=torch.empty_like(x), + ) + if has_lora_delta: + # Only compare rows belonging to real tokens, excluding padding. + indices = result[2].flatten().long() + return result[0][indices], result[3][indices] + return result + + actual = invoke(w1, w2) + expected = invoke(w1.view(128, 4, 256, 64), w2.view(128, 2, 256, 64)) + for actual_tensor, expected_tensor in zip(actual, expected): + torch.testing.assert_close(actual_tensor, expected_tensor) diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py index 446acf46dd..54227d3cd1 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py @@ -31,6 +31,15 @@ from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe +def view_as_block_major_k(weight: torch.Tensor) -> torch.Tensor: + """View packed storage, including legacy 4D IPC cache entries.""" + if weight.ndim == 4: + return weight + experts, rows, cols = weight.shape + block_k = 128 // weight.element_size() + return weight.view(experts, cols // block_k, rows, block_k) + + class TrtLlmBf16ExpertsBase: """BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and monolithic interfaces. @@ -200,8 +209,8 @@ class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular) result = flashinfer.fused_moe.trtllm_bf16_routed_moe( topk_ids=(topk_ids, topk_weights), hidden_states=hidden_states, - gemm1_weights=w1, - gemm2_weights=w2, + gemm1_weights=view_as_block_major_k(w1), + gemm2_weights=view_as_block_major_k(w2), num_experts=global_num_experts, top_k=topk_ids.size(1), n_group=None, @@ -281,8 +290,8 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol routing_logits=router_logits, routing_bias=e_score_correction_bias, hidden_states=hidden_states, - gemm1_weights=w1, - gemm2_weights=w2, + gemm1_weights=view_as_block_major_k(w1), + gemm2_weights=view_as_block_major_k(w2), num_experts=global_num_experts, top_k=self.topk, n_group=num_expert_group, diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_lora_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_lora_moe.py index 4dc5076cf6..1c5698932a 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_lora_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_lora_moe.py @@ -34,6 +34,9 @@ from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRACon from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import ( LoRAExpertsMixin, ) +from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( + view_as_block_major_k, +) from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( TopKWeightAndReduceNoOP, ) @@ -527,8 +530,8 @@ class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase): ret = flashinfer.fused_moe.trtllm_bf16_routed_moe( topk_ids=topk_ids_and_weights, hidden_states=hidden_states, - gemm1_weights=w1, - gemm2_weights=w2, + gemm1_weights=view_as_block_major_k(w1), + gemm2_weights=view_as_block_major_k(w2), gemm1_lora_delta=gemm1_lora_delta, num_experts=global_num_experts, top_k=self.topk, diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index d1437933c3..bcb00eecbb 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( swap_w13_to_w31, ) from vllm.platforms import current_platform +from vllm.utils.math_utils import round_up if TYPE_CHECKING: from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey @@ -349,6 +350,17 @@ def select_unquantized_moe_backend( ) +def unquantized_round_up_hidden_size_and_intermediate_size( + backend: UnquantizedMoeBackend, + hidden_size: int, + intermediate_size: int, +) -> tuple[int, int]: + """Round up dimensions before allocation to satisfy the selected kernel.""" + if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: + intermediate_size = round_up(intermediate_size, 128) + return hidden_size, intermediate_size + + def convert_to_unquantized_kernel_format( unquantized_backend: UnquantizedMoeBackend, moe_config: FusedMoEConfig, @@ -377,13 +389,26 @@ def convert_to_unquantized_kernel_format( ) moe_config.intermediate_size_per_partition = padded_intermediate + # Reloads only overwrite checkpoint slices. An earlier in-place + # permutation can leave nonzero values in the raw padding slots. + unpadded = moe_config.intermediate_size_per_partition_unpadded + assert unpadded is not None + if padded_intermediate > unpadded: + w13_weight[:, unpadded:padded_intermediate].zero_() + if is_act_and_mul: + w13_weight[:, padded_intermediate + unpadded :].zero_() + w2_weight[:, :, unpadded:].zero_() + _cache_permute_indices: dict[torch.Size, torch.Tensor] = {} - w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout( + convert_moe_weights_to_flashinfer_trtllm_block_layout( _cache_permute_indices, w13_weight, w2_weight, is_gated_act_gemm=is_act_and_mul, ) + # Keep checkpoint-shaped parameters for reload/IPC. The experts create + # BlockMajorK views at dispatch without changing the underlying storage. + return w13_weight, w2_weight if ( unquantized_backend == UnquantizedMoeBackend.TRITON diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index a1b2471f5d..e0835f538c 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( convert_to_unquantized_kernel_format, make_unquantized_moe_kernel, select_unquantized_moe_backend, + unquantized_round_up_hidden_size_and_intermediate_size, ) from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, @@ -34,7 +35,6 @@ from vllm.model_executor.utils import ( set_weight_attrs, ) from vllm.platforms import current_platform -from vllm.utils.math_utils import round_up if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts @@ -72,9 +72,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ): unpadded_intermediate = self.moe.intermediate_size_per_partition_unpadded assert unpadded_intermediate is not None - has_intermediate_padding = ( - intermediate_size_per_partition > unpadded_intermediate - ) if self.moe.is_act_and_mul: w13_up_dim = 2 * intermediate_size_per_partition else: @@ -83,22 +80,15 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): assert unpadded_hidden is not None unpadded_up = unpadded_intermediate * (2 if self.moe.is_act_and_mul else 1) # Fused gate_up_proj (column parallel) - w13_data = torch.empty( - num_experts, - w13_up_dim, - hidden_size, - dtype=params_dtype, + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + w13_up_dim, + hidden_size, + dtype=params_dtype, + ), + requires_grad=False, ) - if has_intermediate_padding: - # The checkpoint loader only writes the unpadded slices. Zero the - # tails now so no full-size padding copy is needed after loading. - w13_data[:, unpadded_intermediate:intermediate_size_per_partition].zero_() - if self.moe.is_act_and_mul: - w13_data[ - :, - intermediate_size_per_partition + unpadded_intermediate :, - ].zero_() - w13_weight = torch.nn.Parameter(w13_data, requires_grad=False) layer.register_parameter("w13_weight", w13_weight) set_weight_attrs(w13_weight, extra_weight_attrs) w13_weight.weight_loader_numel = num_experts * unpadded_up * unpadded_hidden @@ -111,15 +101,15 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): set_weight_attrs(w13_bias, extra_weight_attrs) w13_bias.weight_loader_numel = num_experts * unpadded_up # down_proj (row parallel) - w2_data = torch.empty( - num_experts, - hidden_size, - intermediate_size_per_partition, - dtype=params_dtype, + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, ) - if has_intermediate_padding: - w2_data[:, :, unpadded_intermediate:].zero_() - w2_weight = torch.nn.Parameter(w2_data, requires_grad=False) layer.register_parameter("w2_weight", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) w2_weight.weight_loader_numel = ( @@ -147,11 +137,9 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): act_dtype, moe_parallel_config, ) - if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: - intermediate_size_per_partition = round_up( - intermediate_size_per_partition, 128 - ) - return hidden_size, intermediate_size_per_partition + return unquantized_round_up_hidden_size_and_intermediate_size( + self.unquantized_backend, hidden_size, intermediate_size_per_partition + ) def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor: # Pad the weight tensor. This is an optimization on ROCm platform, which @@ -177,21 +165,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): w13: torch.Tensor, w2: torch.Tensor, ) -> None: - w13_parameter_shape = w13.shape - w2_parameter_shape = w2.shape - - if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: - unpadded = self.moe.intermediate_size_per_partition_unpadded - assert unpadded is not None - intermediate = w2.shape[-1] - # Reloads overwrite only checkpoint slices; the previous in-place - # permutation can leave nonzero values in the raw padding slots. - if intermediate > unpadded: - w13[:, unpadded:intermediate].zero_() - if self.moe.is_act_and_mul: - w13[:, intermediate + unpadded :].zero_() - w2[:, :, unpadded:].zero_() - # Shuffle weights to runtime format. w13_new, w2_new = convert_to_unquantized_kernel_format( self.unquantized_backend, @@ -199,10 +172,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): w13_weight=w13, w2_weight=w2, ) - if self.unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: - w13_new = w13_new.view(w13_parameter_shape) - w2_new = w2_new.view(w2_parameter_shape) - # `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__; # On the first call we replace the parameter normally. On subsequent # calls (e.g. RL weight updates that re-trigger @@ -231,25 +200,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): assert self.moe_kernel is not None self.moe_kernel.fused_experts.process_weights_after_loading(layer) - def _kernel_weights( - self, layer: "RoutedExperts" - ) -> tuple[torch.Tensor, torch.Tensor]: - if self.unquantized_backend != UnquantizedMoeBackend.FLASHINFER_TRTLLM: - return layer.w13_weight, layer.w2_weight - - # IPC caches restore tensor shapes/storage, not method attributes. - # BlockMajorK uses 128-byte blocks, i.e. 64 BF16 elements. - def block_view(weight: torch.Tensor) -> torch.Tensor: - if weight.ndim == 4: - return weight - experts, rows, cols = weight.shape - return weight.view(experts, cols // 64, rows, 64) - - return ( - block_view(layer.w13_weight), - block_view(layer.w2_weight), - ) - def _init_moe_kernel(self, layer: "RoutedExperts") -> None: """Build the MoE kernel from the layer's current (shuffled) weights.""" self.moe_quant_config = self.get_fused_moe_quant_config(layer) @@ -354,11 +304,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): shared_experts_input: torch.Tensor | None, ) -> torch.Tensor: assert self.moe_kernel is not None - w13, w2 = self._kernel_weights(layer) return self.moe_kernel.apply( hidden_states=x, - w1=w13, - w2=w2, + w1=layer.w13_weight, + w2=layer.w2_weight, topk_weights=topk_weights, topk_ids=topk_ids, activation=layer.activation, @@ -396,11 +345,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ) -> torch.Tensor | UnfinalizedMoEOutput: assert self.is_monolithic assert self.moe_kernel is not None - w13, w2 = self._kernel_weights(layer) return self.moe_kernel.apply_monolithic( x, - w13, - w2, + layer.w13_weight, + layer.w2_weight, router_logits, activation=layer.activation, global_num_experts=layer.global_num_experts, From e8a2a0a8bbe675885ced6efddd49c1da0ff0a218 Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Thu, 17 Sep 2026 15:02:00 -0700 Subject: [PATCH 061/143] [Bugfix][Frontend] Reject stop strings on --tokens-only servers instead of silently ignoring them (#57058) Signed-off-by: Shimi Bandiel Co-authored-by: Claude Co-authored-by: Cyrus Leung Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../token_in_token_out/test_serving_tokens.py | 25 +++++++++++++++++++ .../scale_out/token_in_token_out/serving.py | 10 ++++++++ 2 files changed, 35 insertions(+) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 1c349e8d67..aa90618a90 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -422,6 +422,31 @@ async def test_stop_string_workflow(client, tokenizer, messages): assert generate_res == completions_res +@pytest.mark.asyncio +@pytest.mark.skipif( + envs.VLLM_USE_RUST_FRONTEND, + reason="--tokens-only is not supported by the Rust frontend", +) +@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True) +async def test_stop_strings_rejected_when_tokens_only(client): + """--tokens-only forces detokenize=False after request validation, so + stop strings must be rejected explicitly rather than silently ignored.""" + sampling_params = {"max_tokens": 5, "stop": ["never"]} + payload = { + "model": MODEL_NAME, + "token_ids": [1, 2, 3], + "sampling_params": sampling_params, + "stream": False, + } + resp = await client.post(GEN_ENDPOINT, json=payload) + assert resp.status_code == 400 + assert "stop strings" in resp.json()["error"]["message"] + + sampling_params["stop"] = None + resp = await client.post(GEN_ENDPOINT, json=payload) + resp.raise_for_status() + + @pytest.mark.asyncio @pytest.mark.parametrize( "server", diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index a8a2804a06..7813c7facc 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -134,6 +134,16 @@ class ServingTokens(GenerateBaseServing): f"sampling_params.n must be at most the server's max_num_seqs " f"({max_num_seqs}), got {sampling_params.n}." ) + if self.force_no_detokenize and sampling_params.stop: + # SamplingParams rejects stop with detokenize=False at request + # validation, but this server forces detokenize=False afterwards, + # so the combination must be rejected here or stop strings are + # silently never applied. + return self.create_error_response( + "stop strings are not supported on a --tokens-only server " + "because detokenization is disabled. Check stop strings on " + "the coordinator, or use stop_token_ids." + ) try: msgspec.msgpack.encode( ( From d3074acddade913b530e192143b3611f58938ce3 Mon Sep 17 00:00:00 2001 From: rasmith Date: Thu, 17 Sep 2026 17:13:13 -0500 Subject: [PATCH 062/143] [AMD][CI][The Rock] Fix language models standard for The Rock on mi355 (#53837) Signed-off-by: Randall Smith Co-authored-by: Andreas Karatzas --- tests/models/language/generation/test_common.py | 10 ++-------- tests/models/registry.py | 5 ++++- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/models/language/generation/test_common.py b/tests/models/language/generation/test_common.py index 33ff4e7ccb..50a0a5ed8c 100644 --- a/tests/models/language/generation/test_common.py +++ b/tests/models/language/generation/test_common.py @@ -26,7 +26,7 @@ AITER_MODEL_LIST = [ "meta-llama/Llama-3.2-1B-Instruct", "openbmb/MiniCPM3-4B", "Qwen/Qwen2.5-0.5B-Instruct", - "TitanML/tiny-mixtral", + "axolotl-ai-co/tiny-mixtral-30m", "Qwen/Qwen3-8B", ] @@ -95,7 +95,7 @@ AITER_MODEL_LIST = [ pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm pytest.param("bigcode/starcoder2-3b"), # starcoder2 pytest.param( - "TitanML/tiny-mixtral", # mixtral + "axolotl-ai-co/tiny-mixtral-30m", # mixtral marks=[pytest.mark.core_model], ), pytest.param("swiss-ai/Apertus-8B-Instruct-2509"), # apertus @@ -126,12 +126,6 @@ def test_models( model_info.check_available_online(on_fail="skip") model_info.check_transformers_version(on_fail="skip") - if current_platform.is_rocm() and model == "TitanML/tiny-mixtral": - # Its single-token router selects LLMM1, whose low-precision - # accumulation can change the top-2 experts. Keep the optimized kernel - # enabled generally, but use the reference GEMM for this accuracy test. - monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0") - if use_rocm_aiter and (model in AITER_MODEL_LIST): monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") elif use_rocm_aiter and model not in AITER_MODEL_LIST: diff --git a/tests/models/registry.py b/tests/models/registry.py index 8adb9ed152..6a0fe8318e 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -447,7 +447,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { ), "MixtralForCausalLM": _HfExamplesInfo( "mistralai/Mixtral-8x7B-Instruct-v0.1", - {"tiny": "TitanML/tiny-mixtral"}, + { + "tiny": "axolotl-ai-co/tiny-mixtral-30m", + "tiny-random": "TitanML/tiny-mixtral", + }, ), "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"), "NemotronHForCausalLM": _HfExamplesInfo( From 80447d27655918da6bfbccd0d3a40e975bda220a Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Thu, 17 Sep 2026 15:14:37 -0700 Subject: [PATCH 063/143] [Bugfix][DSv4.1] Fix FlashInfer DSpark non-causal attention (#57432) Signed-off-by: Woosuk Kwon Co-authored-by: Codex --- .../test_dspark_noncausal_sparse_mla.py | 150 ++++++++++++++++++ .../deepseek_v41/nvidia/flashinfer_sparse.py | 68 +++++++- vllm/v1/attention/backends/mla/sparse_swa.py | 2 + 3 files changed, 214 insertions(+), 6 deletions(-) diff --git a/tests/v1/attention/test_dspark_noncausal_sparse_mla.py b/tests/v1/attention/test_dspark_noncausal_sparse_mla.py index d5c7941314..30c35a730e 100644 --- a/tests/v1/attention/test_dspark_noncausal_sparse_mla.py +++ b/tests/v1/attention/test_dspark_noncausal_sparse_mla.py @@ -604,3 +604,153 @@ def test_dspark_noncausal_differs_from_causal( f"non-causal backend output matches the causal reference " f"(max abs diff={causal_err}); future-pointing indices are not attended to" ) + + +@pytest.mark.parametrize("context_len", [20, 128, 900]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize("num_heads", [16, 64]) +def test_dsv41_flashinfer_dspark_window_matches_reference( + context_len, dtype, num_heads, monkeypatch +): + """Draft queries see the full block without treating padded slots as keys.""" + if not current_platform.is_device_capability_family(100): + pytest.skip("DSV4 TRTLLM sparse attention requires SM100") + from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import ( + DeepseekSparseSWAFlashInferMetadataBuilder, + DeepseekV4FlashInferMLAAttention, + ) + from vllm.models.deepseek_v41.sparse_mla import ( + DeepseekV41SparseSWAMetadataBuilder, + ) + + torch.manual_seed(123) + device = "cuda" + query_lens = [5, 3] + context_lens = [context_len, context_len + 137] + num_real_tokens, num_tokens, head_dim = 8, 9, 512 + cache = torch.randn(32, 128, head_dim, device=device, dtype=torch.bfloat16).to( + dtype + ) + query = torch.randn( + num_tokens, num_heads, head_dim, device=device, dtype=torch.bfloat16 + ).to(dtype) + indices = torch.full((num_tokens, 256), -1, device=device, dtype=torch.int32) + visible_indices = [] + visible_lens = [] + for req, (context, query_len) in enumerate(zip(context_lens, query_lens)): + visible = ( + torch.arange(max(context - 128, 0), context + query_len, device=device) + + req * 16 * 128 + ) + visible_indices.extend([visible] * query_len) + visible_lens.extend([visible.numel()] * query_len) + for token, visible in enumerate(visible_indices): + indices[token, : visible.numel()] = visible.to(torch.int32) + query_start_loc = torch.tensor([0, 5, 8, 9], dtype=torch.int32) + metadata = SimpleNamespace( + num_decodes=3, + num_prefills=0, + num_decode_tokens=num_tokens, + num_prefill_tokens=0, + seq_lens=torch.tensor( + [context_lens[0] + 5, context_lens[1] + 3, 1], + device=device, + dtype=torch.int32, + ), + query_start_loc=query_start_loc.to(device), + query_start_loc_cpu=query_start_loc, + token_to_req_indices=torch.tensor( + [0] * 5 + [1] * 3 + [2], device=device, dtype=torch.int32 + ), + decode_swa_indices=indices, + decode_swa_width=256, + decode_swa_lens=torch.tensor( + visible_lens + [0], device=device, dtype=torch.int32 + ), + block_table=torch.arange(48, device=device, dtype=torch.int32).view(3, -1), + block_size=128, + flashinfer_sparse_index_cache={}, + max_decode_query_len=5, + ) + + # Exercise FlashInfer preparation without constructing a model/config. + def init_parent(builder): + builder._max_tokens = num_tokens + builder.device = device + builder.window_size = 128 + + monkeypatch.setattr(DeepseekV41SparseSWAMetadataBuilder, "__init__", init_parent) + monkeypatch.setattr( + DeepseekV41SparseSWAMetadataBuilder, + "build", + lambda *args: metadata, + ) + builder = DeepseekSparseSWAFlashInferMetadataBuilder() + common_metadata = SimpleNamespace(causal=False) + builder.build(0, common_metadata) + prepared = ( + metadata.flashinfer_decode_topk_lens, + metadata.flashinfer_decode_seq_lens, + ) + attention = SimpleNamespace( + kv_cache_torch_dtype=dtype, + window_size=128, + compress_ratio=0, + topk_indices_buffer=torch.empty( + num_tokens, 0, device=device, dtype=torch.int32 + ), + scale=1 / math.sqrt(head_dim), + _flashinfer_fp8_bmm1_scale=1 / math.sqrt(head_dim), + _flashinfer_fp8_bmm2_scale=1.0, + attn_sink=None, + ) + attention._build_sparse_index_metadata = MethodType( + DeepseekV4FlashInferMLAAttention._build_sparse_index_metadata, attention + ) + output = torch.empty_like(query, dtype=torch.bfloat16) + + def forward(): + metadata.flashinfer_sparse_index_cache.clear() + DeepseekV4FlashInferMLAAttention._forward( + attention, query, None, cache, metadata, None, True, output + ) + + def check_output(): + references = [] + for token, visible in enumerate(visible_indices): + keys = cache.flatten(0, 1)[visible].float() + weights = torch.softmax( + query[token].float() @ keys.T / math.sqrt(head_dim), -1 + ) + references.append(weights @ keys) + atol = 0.01 if dtype == torch.bfloat16 else 0.05 + torch.testing.assert_close( + output[:num_real_tokens].float(), + torch.stack(references), + atol=atol, + rtol=0.05, + ) + + forward() + check_output() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + forward() + # Replay must consume the current inputs, including KV written since capture. + query.copy_(torch.randn_like(query, dtype=torch.bfloat16).to(dtype)) + cache.copy_(torch.randn_like(cache, dtype=torch.bfloat16).to(dtype)) + # A new step changes visibility as well as Q/KV, without recapturing. + metadata.seq_lens.sub_(1) + metadata.decode_swa_lens[:num_real_tokens].sub_(1) + for token, visible in enumerate(visible_indices): + indices[token, visible.numel() - 1] = -1 + visible_indices[token] = visible[:-1] + builder.build(0, common_metadata) + assert metadata.flashinfer_decode_topk_lens.data_ptr() == prepared[0].data_ptr() + assert metadata.flashinfer_decode_seq_lens.data_ptr() == prepared[1].data_ptr() + torch.testing.assert_close(prepared[0], metadata.decode_swa_lens.clamp_min(128)) + torch.testing.assert_close( + prepared[1], metadata.seq_lens[metadata.token_to_req_indices.long()] + ) + graph.replay() + check_output() diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index 409725434a..a247ae7d74 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -26,7 +26,11 @@ from vllm.models.deepseek_v41.sparse_mla import ( ) from vllm.platforms.interface import DeviceCapability from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4 -from vllm.v1.attention.backend import AttentionCGSupport, MultipleOf +from vllm.v1.attention.backend import ( + AttentionCGSupport, + CommonAttentionMetadata, + MultipleOf, +) from vllm.v1.attention.backends.mla.compressor_utils import ( get_dspark_swa_index_width, ) @@ -183,6 +187,39 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Graphs retain these addresses while each build refreshes their contents. + self._decode_topk_lens = torch.empty( + self._max_tokens, dtype=torch.int32, device=self.device + ) + self._decode_seq_lens = torch.empty_like(self._decode_topk_lens) + + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> "DeepseekSparseSWAMetadata": + metadata = super().build(common_prefix_len, common_attn_metadata, fast_build) + num_tokens = metadata.num_decode_tokens + if not common_attn_metadata.causal and num_tokens > 0: + assert metadata.decode_swa_lens is not None + assert metadata.seq_lens is not None + assert metadata.token_to_req_indices is not None + topk_lens = self._decode_topk_lens[:num_tokens] + seq_lens = self._decode_seq_lens[:num_tokens] + torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens) + torch.index_select( + metadata.seq_lens, + 0, + metadata.token_to_req_indices[:num_tokens], + out=seq_lens, + ) + metadata.flashinfer_decode_topk_lens = topk_lens + metadata.flashinfer_decode_seq_lens = seq_lens + return metadata + class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend): @staticmethod @@ -493,21 +530,40 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): # Keep the TRTLLM-gen decode/prefill split: the launcher is tuned for # uniform-q batches, and this avoids flattening mixed batches into one call. if num_decode_tokens > 0: + decode_query = query[:num_decode_tokens] + decode_output = output[:num_decode_tokens] decode_cu = query_start_loc[: num_decodes + 1] + decode_seq_lens = seq_lens[:num_decodes] + decode_topk_lens = sparse_topk_lens[:num_decode_tokens] + max_decode_query_len = swa_metadata.max_decode_query_len + if swa_metadata.decode_swa_width > self.window_size: + # DSpark's non-causal window extends past the fixed 128 SWA + # columns into the aliased compressed pool. Exclude padding, + # and expose the full block to each query instead of letting + # TRTLLM derive a causal SWA length from its query position. + assert swa_only + assert swa_metadata.flashinfer_decode_topk_lens is not None + assert swa_metadata.flashinfer_decode_seq_lens is not None + decode_topk_lens = swa_metadata.flashinfer_decode_topk_lens + decode_seq_lens = swa_metadata.flashinfer_decode_seq_lens + decode_query = decode_query.unsqueeze(1) + decode_output = decode_output.unsqueeze(1) + decode_cu = None + max_decode_query_len = 1 flashinfer_trtllm_batch_decode_sparse_mla_dsv4( - query=query[:num_decode_tokens], + query=decode_query, swa_kv_cache=swa_k_cache, workspace_buffer=workspace, sparse_indices=sparse_indices[:num_decode_tokens], compressed_kv_cache=compressed_kv_cache, - sparse_topk_lens=sparse_topk_lens[:num_decode_tokens], - seq_lens=seq_lens[:num_decodes], - out=output[:num_decode_tokens], + sparse_topk_lens=decode_topk_lens, + seq_lens=decode_seq_lens, + out=decode_output, bmm1_scale=bmm1_scale, bmm2_scale=bmm2_scale, sinks=self.attn_sink, cum_seq_lens_q=decode_cu, - max_q_len=swa_metadata.max_decode_query_len, + max_q_len=max_decode_query_len, ) if num_prefill_tokens > 0: diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index e88631c32b..0fa5ae414c 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -205,6 +205,8 @@ class DeepseekSparseSWAMetadata: num_decode_tokens: int = 0 num_prefill_tokens: int = 0 max_decode_query_len: int = 1 + flashinfer_decode_topk_lens: torch.Tensor | None = None + flashinfer_decode_seq_lens: torch.Tensor | None = None # Pre-computed prefill metadata shared across all DeepseekV4 attention layers. prefill_seq_lens: torch.Tensor | None = None From e0050f287aae8b3ddbe2947a5af1c5dd23db2781 Mon Sep 17 00:00:00 2001 From: Mustafa YILDIRIM <2341474+mustafayildirim@users.noreply.github.com> Date: Thu, 17 Sep 2026 15:34:56 -0700 Subject: [PATCH 064/143] [Bugfix][ROCm] Add record_logical_topk_ready to ROCMAiterMLASparseImpl (GLM-5.3-Flash boot crash) (#57252) Signed-off-by: Mustafa YILDIRIM Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Andreas Karatzas --- vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index e1365f4d30..69b6e342cd 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -808,6 +808,11 @@ class ROCMAiterMLASparseImpl( (q_concat_shape, vllm_config.model_config.dtype), ) + def record_logical_topk_ready(self) -> None: + # This impl shares the top-k indices buffer via SharedTopkIndicesBuffer + # but does not participate in sparse-MLA index groups. + pass + def _forward_mla( self, layer: AttentionLayer, From ccde9c61e958d9dbfc21af6c62df6ff15881c811 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:01:35 -0400 Subject: [PATCH 065/143] [Bugfix] Restore KV cache metadata GET method for external event consumer (#56925) Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> --- .../v1/engine/test_kv_cache_group_metadata.py | 70 +++++++++++++++++++ vllm/v1/engine/core.py | 32 ++++++++- 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 tests/v1/engine/test_kv_cache_group_metadata.py diff --git a/tests/v1/engine/test_kv_cache_group_metadata.py b/tests/v1/engine/test_kv_cache_group_metadata.py new file mode 100644 index 0000000000..1b2dedec4d --- /dev/null +++ b/tests/v1/engine/test_kv_cache_group_metadata.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.v1.core.kv_cache_manager import KVCacheManager +from vllm.v1.engine.core import EngineCore +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + KVCacheGroupSpec, + MambaSpec, + MLAAttentionSpec, +) + +pytestmark = pytest.mark.cpu_test + +BASE_BLOCK_SIZE = 1536 +DCP_WORLD_SIZE = 8 + + +def _make_engine_core_with_dcp_manager() -> EngineCore: + mla_spec = MLAAttentionSpec( + block_size=BASE_BLOCK_SIZE, + num_kv_heads=1, + head_size=576, + dtype=torch.bfloat16, + ) + mamba_spec = MambaSpec( + block_size=BASE_BLOCK_SIZE, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + config = KVCacheConfig( + num_blocks=32, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["mla"], mla_spec), + KVCacheGroupSpec(["mamba"], mamba_spec), + ], + ) + manager = KVCacheManager( + config, + max_model_len=BASE_BLOCK_SIZE * DCP_WORLD_SIZE * 2, + scheduler_block_size=BASE_BLOCK_SIZE * DCP_WORLD_SIZE, + hash_block_size=BASE_BLOCK_SIZE, + dcp_world_size=DCP_WORLD_SIZE, + ) + scheduler = SimpleNamespace(kv_cache_config=config, kv_cache_manager=manager) + engine_core = object.__new__(EngineCore) + engine_core.scheduler = scheduler + return engine_core + + +def test_kv_cache_group_metadata_uses_effective_dcp_block_sizes() -> None: + engine_core = _make_engine_core_with_dcp_manager() + + metadata = engine_core.get_kv_cache_group_metadata() + + # DCP scales attention blocks but does not scale replicated Mamba state. + assert [item["block_size"] for item in metadata] == [12288, 1536] + assert [item["kind"] for item in metadata] == ["mla_attention", "mamba"] + + managers = engine_core.scheduler.kv_cache_manager.coordinator.single_type_managers + assert [item["block_size"] for item in metadata] == [ + manager.block_size for manager in managers + ] diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 7da7f2e0ec..14224f95e0 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -85,7 +85,11 @@ from vllm.v1.fault_tolerance.engine_core_sentinel import ( EngineCoreSentinel, fault_tolerant_wrapper, ) -from vllm.v1.kv_cache_interface import KVCacheConfig, is_full_attention_spec +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + get_kv_cache_spec_kind, + is_full_attention_spec, +) from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats from vllm.v1.outputs import ModelRunnerOutput from vllm.v1.request import Request, RequestStatus @@ -457,6 +461,32 @@ class EngineCore: supported_pooling_tasks, ) + def get_kv_cache_group_metadata(self) -> list[dict[str, int | str | None]]: + """Return serializable KV cache metadata for external event consumers + (e.g., Dynamo). This function has no call sites within vLLM but needs to + be kept. + """ + kv_cache_config = getattr(self.scheduler, "kv_cache_config", None) + if kv_cache_config is None: + return [] + + kv_cache_manager = cast(Any, self.scheduler).kv_cache_manager + managers = kv_cache_manager.coordinator.single_type_managers + metadata: list[dict[str, int | str | None]] = [] + for group_idx, (group, manager) in enumerate( + zip(kv_cache_config.kv_cache_groups, managers, strict=True) + ): + spec = group.kv_cache_spec + metadata.append( + { + "group_idx": group_idx, + "kind": get_kv_cache_spec_kind(spec).value, + "block_size": manager.block_size, + "sliding_window": getattr(spec, "sliding_window", None), + } + ) + return metadata + def add_request(self, request: Request, request_wave: int = 0): """Add request to the scheduler. From a524f80b1f0dd85889bf227310670ee8fdcd5fd2 Mon Sep 17 00:00:00 2001 From: Oxana Korzh Date: Thu, 17 Sep 2026 17:09:06 -0600 Subject: [PATCH 066/143] [AMD][Bugfix] Make the nested-RoPE patch reach automatic validation (#57289) Signed-off-by: Oxana Korzh Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Andreas Karatzas --- vllm/transformers_utils/config.py | 48 ++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 20d0d73afb..e78a718e28 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -162,11 +162,12 @@ _SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"} _PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"} -# Model types whose checkpoints store shared RoPE parameters alongside the -# per-layer-type dicts (e.g. Laguna's `original_max_position_embeddings`). -# Since transformers 5.17, `validate_rope` treats every top-level value of such -# a dict as a layer's parameters and raises on the shared ones. -_PATCH_HF_NESTED_ROPE_VALIDATION: set[str] = {"laguna"} +# Model types whose checkpoints carry shared RoPE parameters alongside the +# per-layer-type dicts. Since transformers 5.17, `validate_rope` treats every +# top-level value of such a dict as a layer's parameters and raises on the +# shared ones. `laguna` gets them injected by `convert_rope_params_to_dict`; +# `gemma4_text` ships them in the checkpoint itself. +_PATCH_HF_NESTED_ROPE_VALIDATION: set[str] = {"laguna", "gemma4_text"} # Model types whose checkpoints declare `layer_types` entries that upstream # transformers has not added to `ALLOWED_LAYER_TYPES` yet, so its strict config @@ -224,6 +225,35 @@ def _mistral_patch_hf_hub_constants() -> Iterator[None]: constants.SAFETENSORS_INDEX_FILE = hf_safetensors_index_file +def _install_hf_config_validator( + name: str, validator: Callable, supersedes: Callable +) -> None: + """Replace a ``PretrainedConfig`` validator on every ``@strict`` snapshot. + + ``@strict`` snapshots each ``validate_*`` method into ``__class_validators__`` + at class creation and automatic post-``__init__`` validation dispatches off + that frozen list, so assigning the class attribute alone only affects + explicit ``config.validate_*()`` calls. Every ``@strict``-decorated config + class owns a snapshot, so rewrite them all, matching ``supersedes`` by + identity to leave a genuine per-model override in place. + """ + setattr(PretrainedConfig, name, validator) + + seen: set[int] = set() + stack = [PretrainedConfig] + while stack: + cls = stack.pop() + if id(cls) in seen: + continue + seen.add(id(cls)) + validators = cls.__dict__.get("__class_validators__") + if isinstance(validators, list) and any(v is supersedes for v in validators): + cls.__class_validators__ = [ + validator if v is supersedes else v for v in validators + ] + stack.extend(cls.__subclasses__()) + + def _patch_hf_transformers_validate_rope(): """Transformers v5 moved the ignore_keys option from the method signature of validate_rope and replaced it with the ignore_keys_at_rope_validation parameter @@ -250,7 +280,9 @@ def _patch_hf_transformers_validate_rope(): return result patched_validate_rope.__vllm_patched__ = True # type: ignore[attr-defined] - PretrainedConfig.validate_rope = patched_validate_rope + _install_hf_config_validator( + "validate_rope", patched_validate_rope, _original_validate_rope + ) def _patch_hf_transformers_nested_rope_validation() -> None: @@ -284,7 +316,9 @@ def _patch_hf_transformers_nested_rope_validation() -> None: return _original_validate_rope(self, *args, **kwargs) patched_validate_rope.__vllm_nested_rope_patched__ = True # type: ignore[attr-defined] - PretrainedConfig.validate_rope = patched_validate_rope + _install_hf_config_validator( + "validate_rope", patched_validate_rope, _original_validate_rope + ) def _patch_hf_transformers_allowed_layer_types( From 092bdd6d57ac7c1cd5272339c80372053fd51bbe Mon Sep 17 00:00:00 2001 From: Dakai An <77474977+andakai@users.noreply.github.com> Date: Fri, 18 Sep 2026 07:12:12 +0800 Subject: [PATCH 067/143] [Frontend][Core] Add release_kv_cache_memory() API (#44890) Signed-off-by: Dakai An Signed-off-by: AlanFokCo Signed-off-by: aoshen02 Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo Co-authored-by: aoshen02 Co-authored-by: OpenAI Codex --- docs/features/sleep_mode.md | 21 +++++ rust/src/engine-core-client/src/client.rs | 17 +++- rust/src/mock-engine/src/engine.rs | 10 +-- rust/src/mock-engine/src/tests.rs | 1 + rust/src/server/src/middleware/metrics.rs | 1 + rust/src/server/src/routes.rs | 4 + rust/src/server/src/routes/sleep.rs | 37 +++++--- rust/src/server/src/routes/tests.rs | 89 ++++++++++++++++++- tests/basic_correctness/test_mem.py | 85 ++++++++++++++++++ tests/entrypoints/serve/dev/test_sleep.py | 25 +++++- tests/multimodal/test_cache.py | 75 ++++++++++++++++ tests/v1/core/test_scheduler.py | 76 ++++++++++++++++ tests/v1/engine/test_engine_core.py | 24 +++++ vllm/device_allocator/sleep_mode_backend.py | 14 +++ vllm/engine/protocol.py | 5 ++ vllm/entrypoints/llm.py | 8 ++ .../entrypoints/serve/dev/sleep/api_router.py | 6 ++ vllm/v1/engine/async_llm.py | 11 ++- vllm/v1/engine/core.py | 29 +++++- vllm/v1/engine/core_client.py | 31 +++++-- vllm/v1/engine/llm_engine.py | 11 ++- vllm/v1/executor/abstract.py | 32 +++++-- vllm/v1/worker/cpu_worker.py | 3 + vllm/v1/worker/gpu_worker.py | 3 + 24 files changed, 574 insertions(+), 44 deletions(-) diff --git a/docs/features/sleep_mode.md b/docs/features/sleep_mode.md index dcb6ad8c94..2616f374ea 100644 --- a/docs/features/sleep_mode.md +++ b/docs/features/sleep_mode.md @@ -74,6 +74,18 @@ llm.wake_up(tags=["weights"]) llm.wake_up(tags=["kv_cache"]) ``` +#### Release only KV cache memory + +`LLM.release_kv_cache_memory()` discards KV cache physical memory while keeping model weights resident. It requires a completed pause and all executor memory to be resident: full sleep, partial wake-up, and repeated release without restoring memory are rejected. Requests retained with `mode="keep"` are recomputed after wake-up. + +Use the default `cumem` backend with managed allocations: `enable_cumem_allocator=True` on CUDA/ROCm (also enabled by `enable_sleep_mode=True`), or `enable_sleep_mode=True` on XPU. Other backends must implement selective discard; CPU and unmanaged allocations are unsupported. + +```python +llm.sleep(level=0, mode="keep") # Wait for the pause to complete. +llm.release_kv_cache_memory() +llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling. +``` + ### Online Serving To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server. @@ -107,9 +119,18 @@ curl -X POST 'http://localhost:8000/collective_rpc' -H 'Content-Type: applicatio curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache' ``` +To release only KV cache memory, wait for the level 0 pause to complete before calling the release endpoint. The same resident-memory and backend requirements as the Python API apply. + +```bash +curl -X POST 'http://localhost:8000/sleep?level=0&mode=keep' +curl -X POST 'http://localhost:8000/release_kv_cache_memory' +curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache' +``` + #### HTTP endpoints - `POST /sleep?level=1` — Put the model to sleep (`level=1`). +- `POST /release_kv_cache_memory` — Discard KV cache memory after a completed pause, while all executor memory is resident. - `POST /wake_up` — Wake up the model. Supports optional `tags` query parameters for partial wake-up (e.g., `?tags=weights`). - `POST /collective_rpc` — Perform a collective remote procedure call (RPC). - `GET /is_sleeping` — Check if the model is sleeping. diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 4cd7420e56..aa6aacf0e0 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -911,13 +911,22 @@ impl EngineCoreClient { Ok(()) } - /// Wake the engine from sleep, optionally limiting the wake-up to specific - /// tags. - pub async fn wake_up(&self, tags: Option>) -> Result<()> { - self.call_utility::<(), _>("wake_up", (tags,)).await?; + /// Release KV cache memory while keeping model weights resident. + pub async fn release_kv_cache_memory(&self) -> Result<()> { + self.call_utility::<(), _>("release_kv_cache_memory", ()).await?; Ok(()) } + /// Wake the engine from sleep, optionally limiting the wake-up to specific + /// tags, and return whether every engine is fully awake. + pub async fn wake_up(&self, tags: Option>) -> Result { + Ok(self + .call_utility::("wake_up", (tags,)) + .await? + .into_iter() + .all(|fully_awake| fully_awake)) + } + /// Pause the scheduler so generation can be halted pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> { self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?; diff --git a/rust/src/mock-engine/src/engine.rs b/rust/src/mock-engine/src/engine.rs index 38f9b21be1..4844aba120 100644 --- a/rust/src/mock-engine/src/engine.rs +++ b/rust/src/mock-engine/src/engine.rs @@ -95,12 +95,10 @@ fn utility_response( "get_supported_tasks" => utility_envelope(vec!["generate"]), "is_sleeping" => utility_envelope(false), "reset_prefix_cache" => utility_envelope(true), - "reset_mm_cache" - | "reset_encoder_cache" - | "profile" - | "sleep" - | "wake_up" - | "execute_dummy_batch" => utility_envelope(()), + "wake_up" => utility_envelope(true), + "reset_mm_cache" | "reset_encoder_cache" | "profile" | "sleep" | "execute_dummy_batch" => { + utility_envelope(()) + } _ => utility_envelope(Value::Nil), }?; diff --git a/rust/src/mock-engine/src/tests.rs b/rust/src/mock-engine/src/tests.rs index bee710fd0c..c1cfbb49cb 100644 --- a/rust/src/mock-engine/src/tests.rs +++ b/rust/src/mock-engine/src/tests.rs @@ -195,6 +195,7 @@ async fn utility_requests_return_minimal_success_responses() { assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache")); client.reset_mm_cache().await.expect("reset mm cache"); client.reset_encoder_cache().await.expect("reset encoder cache"); + assert!(client.wake_up(Some(vec!["kv_cache".to_string()])).await.expect("wake up")); shutdown_mock(client, shutdown, task).await; } diff --git a/rust/src/server/src/middleware/metrics.rs b/rust/src/server/src/middleware/metrics.rs index 1e6a79a556..de07620cf7 100644 --- a/rust/src/server/src/middleware/metrics.rs +++ b/rust/src/server/src/middleware/metrics.rs @@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[ "/reset_encoder_cache", "/collective_rpc", "/sleep", + "/release_kv_cache_memory", "/wake_up", "/is_sleeping", ]; diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 41e4987882..1cd6af3d4f 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -142,6 +142,10 @@ fn build_router_with_options( .route("/weight_info", get(weight_transfer::weight_info)) .route("/abort_requests", post(abort_requests::abort_requests)) .route("/sleep", post(sleep::sleep)) + .route( + "/release_kv_cache_memory", + post(sleep::release_kv_cache_memory), + ) .route("/wake_up", post(sleep::wake_up)) .route("/is_sleeping", get(sleep::is_sleeping)) .route("/pause", post(pause::pause)) diff --git a/rust/src/server/src/routes/sleep.rs b/rust/src/server/src/routes/sleep.rs index 4bf9981402..d64a0850b6 100644 --- a/rust/src/server/src/routes/sleep.rs +++ b/rust/src/server/src/routes/sleep.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use axum::Json; use axum::extract::rejection::QueryRejection; -use axum::extract::{Query, State}; +use axum::extract::{Query, RawQuery, State}; use axum::http::StatusCode; use serde::{Deserialize, Serialize}; use vllm_engine_core_client::protocol::utility::PauseMode; @@ -27,12 +27,6 @@ pub(crate) struct SleepParams { mode: PauseMode, } -#[derive(Debug, Default, Deserialize)] -pub(crate) struct WakeUpParams { - #[serde(default)] - tags: Option>, -} - const fn default_sleep_level() -> u32 { 1 } @@ -57,14 +51,35 @@ pub async fn sleep( Ok(StatusCode::OK) } -/// Wake the engine from sleep mode. -pub async fn wake_up( +/// Release KV cache memory while keeping model weights resident. +pub async fn release_kv_cache_memory( State(state): State>, - Query(params): Query, ) -> Result { state .engine_core_client() - .wake_up(params.tags) + .release_kv_cache_memory() + .await + .map_err(|error| utility_call_error("release_kv_cache_memory", error))?; + + Ok(StatusCode::OK) +} + +/// Wake the engine from sleep mode. +pub async fn wake_up( + State(state): State>, + RawQuery(query): RawQuery, +) -> Result { + let tags = query.and_then(|query| { + let tags = url::form_urlencoded::parse(query.as_bytes()) + .filter(|(key, _)| key == "tags") + .map(|(_, value)| value.into_owned()) + .collect::>(); + (!tags.is_empty()).then_some(tags) + }); + + state + .engine_core_client() + .wake_up(tags) .await .map_err(|error| utility_call_error("wake_up", error))?; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 4731958e2f..c27ac20fad 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -5805,7 +5805,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] -async fn wake_up_route_without_tags_sends_none() { +async fn release_kv_cache_memory_route_sends_expected_utility_call() { + let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + + assert_eq!(array[2], Value::from("release_kv_cache_memory")); + assert_eq!(array[3], Value::Array(Vec::new())); + + send_outputs(push, utility_outputs(call_id, utility_none_result())).await; + }) + }) + .await; + + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/release_kv_cache_memory") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert!(body.is_empty()); + engine_task.await.expect("mock engine task"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn wake_up_route_without_tags_accepts_bool_result() { let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -5818,7 +5857,7 @@ async fn wake_up_route_without_tags_sends_none() { assert_eq!(array[2], Value::from("wake_up")); assert_eq!(array[3], Value::Array(vec![Value::Nil])); - send_outputs(push, utility_outputs(call_id, utility_none_result())).await; + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; }) }) .await; @@ -5842,6 +5881,51 @@ async fn wake_up_route_without_tags_sends_none() { engine_task.await.expect("mock engine task"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn wake_up_route_accepts_repeated_tags() { + let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + + assert_eq!(array[2], Value::from("wake_up")); + assert_eq!( + array[3], + Value::Array(vec![Value::Array(vec![ + Value::from("weights"), + Value::from("kv_cache"), + ])]) + ); + + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + }) + }) + .await; + + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert!(body.is_empty()); + engine_task.await.expect("mock engine task"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn is_sleeping_route_returns_json_payload() { @@ -6160,6 +6244,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() { for (method, uri) in [ ("GET", "/is_sleeping"), ("POST", "/sleep"), + ("POST", "/release_kv_cache_memory"), ("POST", "/wake_up"), ("GET", "/is_paused"), ("POST", "/pause"), diff --git a/tests/basic_correctness/test_mem.py b/tests/basic_correctness/test_mem.py index 6b4895c643..746838dc40 100644 --- a/tests/basic_correctness/test_mem.py +++ b/tests/basic_correctness/test_mem.py @@ -109,6 +109,91 @@ def test_basic_cumem(): assert torch.allclose(output, torch.ones_like(output) * 3) +@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch): + """Rejected release and KV-only/full sleep cycles preserve greedy tokens.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + kv_cache_memory_bytes = 256 * 1024 * 1024 + llm = LLM( + "Qwen/Qwen3-0.6B", + enable_sleep_mode=True, + enforce_eager=True, + kv_cache_memory_bytes=kv_cache_memory_bytes, + max_model_len=1024, + max_num_seqs=4, + gpu_memory_utilization=0.05, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + + def get_mapped_bytes(worker): + return mapped_usage(get_mem_allocator_instance()) + + mapped_before = llm.collective_rpc(get_mapped_bytes)[0] + # Utility RPC transports the engine error as a plain Exception. + with pytest.raises(Exception, match="requires a completed pause"): + llm.release_kv_cache_memory() + assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before + assert not llm.llm_engine.is_sleeping() + assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected + + llm.sleep(level=0) + llm.release_kv_cache_memory() + mapped_after = llm.collective_rpc(get_mapped_bytes)[0] + assert mapped_before - mapped_after >= kv_cache_memory_bytes * 0.99 + assert mapped_after > 0 + assert llm.llm_engine.is_sleeping() + + if full_sleep: + llm.sleep(level=full_sleep) + assert llm.collective_rpc(get_mapped_bytes)[0] == 0 + llm.wake_up(tags=None if full_sleep else ["kv_cache"]) + if full_sleep == 2: + llm.collective_rpc("reload_weights") + assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before + assert not llm.llm_engine.is_sleeping() + actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + assert actual == expected + + +@pytest.mark.parametrize("first_level", [1, 2]) +@pytest.mark.parametrize("second_level", [1, 2]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch): + """Repeated sleep after KV-only wake preserves mappings and recoverability.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + llm = LLM( + "Qwen/Qwen3-0.6B", + enable_sleep_mode=True, + enforce_eager=True, + kv_cache_memory_bytes=256 * 1024 * 1024, + max_model_len=1024, + max_num_seqs=4, + gpu_memory_utilization=0.05, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + + def get_mapped_bytes(worker): + return mapped_usage(get_mem_allocator_instance()) + + llm.sleep(level=first_level) + llm.wake_up(tags=["kv_cache"]) + mapped_before = llm.collective_rpc(get_mapped_bytes)[0] + assert mapped_before > 0 + llm.sleep(level=second_level) + assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before + llm.wake_up(tags=["weights"]) + if first_level == 2: + llm.collective_rpc("reload_weights") + assert not llm.llm_engine.is_sleeping() + actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + assert actual == expected + + @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") def test_discard_tags(): """Test that discard(tags) selectively frees GPU memory for specific diff --git a/tests/entrypoints/serve/dev/test_sleep.py b/tests/entrypoints/serve/dev/test_sleep.py index 5ebee50600..34e69c9ece 100644 --- a/tests/entrypoints/serve/dev/test_sleep.py +++ b/tests/entrypoints/serve/dev/test_sleep.py @@ -1,14 +1,37 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import AsyncMock + +import pytest import requests +from fastapi import FastAPI +from fastapi.testclient import TestClient from prometheus_client.parser import text_string_to_metric_families from tests.utils import RemoteOpenAIServer +from vllm.entrypoints.serve.dev.sleep.api_router import attach_router MODEL_NAME = "meta-llama/Llama-3.2-1B" +@pytest.mark.cpu_test +@pytest.mark.parametrize("fails", [False, True]) +def test_release_kv_cache_memory_route(fails): + app = FastAPI() + app.state.engine_client = AsyncMock() + release = app.state.engine_client.release_kv_cache_memory + if fails: + release.side_effect = RuntimeError("requires a completed pause first") + attach_router(app) + + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post("/release_kv_cache_memory") + + assert response.status_code == (500 if fails else 200) + release.assert_awaited_once_with() + + def test_sleep_mode(): # dtype, max-len etc set so that this can run in CI args = [ @@ -63,7 +86,7 @@ def test_sleep_mode(): ) assert response.status_code == 200 - # is sleeping should be false after waking up any part of the engine + # Partial wake keeps the engine sleeping. response = requests.get(remote_server.url_for("is_sleeping")) assert response.status_code == 200 assert response.json().get("is_sleeping") is True diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index f77dacd341..0c68d2bac0 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import multiprocessing as mp +from concurrent.futures import ThreadPoolExecutor +from functools import partial from types import SimpleNamespace +from unittest.mock import Mock import numpy as np import pytest @@ -34,6 +37,8 @@ from vllm.multimodal.inputs import ( PlaceholderRange, ) from vllm.multimodal.processing import PromptInsertion +from vllm.renderers.base import BaseRenderer +from vllm.utils.async_utils import make_async from vllm.utils.mem_constants import GiB_bytes, MiB_bytes from ..utils import create_new_process_for_each_test @@ -755,6 +760,76 @@ def test_processor_cache_shared_across_loras(): assert feature_lora_b.data == item_data +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize( + "release_error", + [ + None, + "requires a completed pause first", + "requires all executor memory to be resident", + ], + ids=["released", "not-paused", "nonresident-memory"], +) +@pytest.mark.asyncio +async def test_release_kv_cache_resends_mm_payload(use_async, release_error): + """Release must not leave a sender hit pointing at a cleared receiver.""" + from vllm.v1.engine.async_llm import AsyncLLM + from vllm.v1.engine.llm_engine import LLMEngine + + model_config = SimpleNamespace( + get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1) + ) + sender = MultiModalProcessorSenderCache(model_config) + receiver = MultiModalReceiverCache(model_config) + item = _dummy_item({"pixel_values": 16}) + mm_hash = "image_A" + payload, _ = sender.get_and_update_item((item, []), mm_hash) + assert receiver.get_and_update_item(payload, mm_hash) is item + assert sender.get_and_update_item(None, mm_hash)[0] is None + + renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None) + renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer) + + def release(): + if release_error: + raise RuntimeError(release_error) + receiver.clear_cache() + + with ThreadPoolExecutor(max_workers=1) as executor: + renderer._clear_mm_cache_async = make_async( + renderer.clear_mm_cache, executor=executor + ) + renderer.clear_mm_cache_async = partial( + BaseRenderer.clear_mm_cache_async, renderer + ) + engine = SimpleNamespace( + renderer=renderer, + engine_core=SimpleNamespace( + release_kv_cache_memory=release, + release_kv_cache_memory_async=make_async(release, executor=executor), + ), + logger_manager=Mock(), + ) + + async def call_release(): + if use_async: + await AsyncLLM.release_kv_cache_memory(engine) + else: + LLMEngine.release_kv_cache_memory(engine) + + if release_error: + with pytest.raises(RuntimeError, match=release_error): + await call_release() + engine.logger_manager.record_sleep_state.assert_not_called() + else: + await call_release() + engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0) + + payload, _ = sender.get_and_update_item((item, []), mm_hash) + assert payload is item + assert receiver.get_and_update_item(payload, mm_hash) is item + + _SLEEP_VISION_PROMPT = ( "<|im_start|>system\nYou are a helpful assistant.<|im_end|>" "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 298c8c6185..4951014d1b 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -36,10 +36,13 @@ from vllm.utils.hashing import sha256 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash +from vllm.v1.core.sched.interface import PauseState from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs from vllm.v1.engine import FinishReason +from vllm.v1.engine.core import EngineCore +from vllm.v1.executor.uniproc_executor import UniProcExecutor from vllm.v1.hisparse.coordinator import get_hisparse_coordinator from vllm.v1.kv_cache_interface import ( FullAttentionSpec, @@ -1487,6 +1490,79 @@ def test_scheduler_reset_prefix_cache(): assert scheduler.waiting[i] == request +def test_kv_cache_release_after_keep_pause_preserves_requests(): + """A completed keep pause permits release and recomputation of live requests.""" + scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True) + running, waiting = create_requests(num_requests=2) + for request in (running, waiting): + scheduler.add_request(request) + scheduler_output = scheduler.schedule() + scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput( + req_ids=[running.request_id], + req_id_to_index={running.request_id: 0}, + sampled_token_ids=[[1]], + ), + ) + assert scheduler.kv_cache_manager.get_block_ids(running.request_id)[0] + + core = object.__new__(EngineCore) + core.scheduler = scheduler + core.model_executor = Mock(is_sleeping=False) + core.mm_receiver_cache = None + core.batch_queue = None + + assert core.pause_scheduler(mode="keep", clear_cache=False) is None + assert scheduler.running == [running] + assert list(scheduler.waiting) == [waiting] + assert not scheduler.has_requests() + + core.release_kv_cache_memory() + + core.model_executor.discard.assert_called_once_with(("kv_cache",)) + assert scheduler.requests == { + request.request_id: request for request in (running, waiting) + } + assert not scheduler.running + assert list(scheduler.waiting) == [running, waiting] + assert running.status == RequestStatus.PREEMPTED + assert running.num_computed_tokens == 0 + assert list(running.output_token_ids) == [1] + assert not scheduler.kv_cache_manager.get_block_ids(running.request_id)[0] + assert scheduler.schedule().total_num_scheduled_tokens == 0 + + assert core.wake_up(tags=["kv_cache"]) + resumed = scheduler.schedule() + assert resumed.num_scheduled_tokens == {running.request_id: running.num_tokens} + + +@pytest.mark.parametrize( + "sleeping_tags", + [{"weights", "kv_cache"}, {"weights"}, {"kv_cache"}], + ids=["full-sleep", "weights-only", "kv-only-or-repeated-release"], +) +def test_kv_cache_release_rejects_nonresident_memory(sleeping_tags): + """Reject release without resetting caches or discarding more memory.""" + core = object.__new__(EngineCore) + core.scheduler = create_scheduler() + core.scheduler.set_pause_state(PauseState.PAUSED_ALL) + core.batch_queue = None + core._reset_caches = Mock() + core.model_executor = object.__new__(UniProcExecutor) + core.model_executor.sleeping_tags = sleeping_tags.copy() + core.model_executor.collective_rpc = Mock() + + with pytest.raises( + RuntimeError, match="requires all executor memory to be resident" + ): + core.release_kv_cache_memory() + + core._reset_caches.assert_not_called() + core.model_executor.collective_rpc.assert_not_called() + assert core.model_executor.sleeping_tags == sleeping_tags + + def test_reset_connector_cache_no_connector_is_no_op_success(): """``reset_connector_cache`` must return True when no connector is configured. diff --git a/tests/v1/engine/test_engine_core.py b/tests/v1/engine/test_engine_core.py index ba1e9a4d20..e16e0550c2 100644 --- a/tests/v1/engine/test_engine_core.py +++ b/tests/v1/engine/test_engine_core.py @@ -23,6 +23,7 @@ from vllm.config import ( from vllm.engine.arg_utils import EngineArgs from vllm.platforms import current_platform from vllm.utils.torch_utils import set_default_torch_num_threads +from vllm.v1.core.sched.interface import PauseState from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc from vllm.v1.executor.abstract import Executor @@ -667,6 +668,29 @@ def _pausable_engine_core_proc() -> EngineCoreProc: return core +@pytest.mark.parametrize( + "pause_state,has_requests,has_batches", + [ + pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"), + pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"), + pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"), + ], +) +def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches): + """Reject release before touching caches or memory if work can still use KV.""" + core = _pausable_engine_core_proc() + core.scheduler.pause_state = pause_state + core.scheduler.has_requests.return_value = has_requests + core.batch_queue = [object()] if has_batches else None + core._reset_caches = MagicMock() + + with pytest.raises(RuntimeError, match="requires a completed pause"): + core.release_kv_cache_memory() + + core._reset_caches.assert_not_called() + core.model_executor.discard.assert_not_called() + + @pytest.mark.parametrize("deferred", [False, True]) def test_pause_synchronizes_device_before_cache_reset(deferred: bool): """A resolved pause promises an idle device: the barrier must run before diff --git a/vllm/device_allocator/sleep_mode_backend.py b/vllm/device_allocator/sleep_mode_backend.py index 45a635d613..023af04275 100644 --- a/vllm/device_allocator/sleep_mode_backend.py +++ b/vllm/device_allocator/sleep_mode_backend.py @@ -73,6 +73,13 @@ class SleepModeBackend(ABC): (suspended) engine from a healthy-serving one (see RFC #34303).""" return self._state + def discard(self, tags: tuple[str, ...]) -> None: + """Discard selected tagged allocations without preserving contents.""" + raise RuntimeError( + f"Sleep-mode backend {type(self).__name__!r} does not support " + "selective discard." + ) + # -- Capability introspection (no instance required) -- @classmethod @@ -132,6 +139,13 @@ class CuMemBackend(SleepModeBackend): allocator.wake_up(tags) self._state = "RUNNING" + def discard(self, tags: tuple[str, ...]) -> None: + from vllm.device_allocator import get_mem_allocator_instance + + allocator = get_mem_allocator_instance() + allocator.discard(tags) + self._state = "SUSPENDED" + @classmethod def preserves_communicators(cls) -> bool: # Comm identity/topology survive memory suspension; no reinit needed. diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index c44fd44d0f..602e2570a2 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -186,6 +186,11 @@ class EngineClient(ABC): """Sleep the engine.""" ... + @abstractmethod + async def release_kv_cache_memory(self) -> None: + """Discard KV cache physical GPU memory. Requires a completed pause.""" + ... + @abstractmethod async def wake_up(self, tags: list[str] | None = None) -> None: """Wake up the engine.""" diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index d9a2a46701..999310aa61 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -830,6 +830,14 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): """ self.llm_engine.sleep(level=level, mode=mode) + def release_kv_cache_memory(self) -> None: + """Release the GPU physical memory backing the KV cache. + + Requires a completed ``sleep(level=0)`` and resident executor memory. + Restore with ``wake_up(tags=["kv_cache"])``; kept requests are recomputed. + """ + self.llm_engine.release_kv_cache_memory() + def wake_up(self, tags: list[str] | None = None): """Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep] method for more details. diff --git a/vllm/entrypoints/serve/dev/sleep/api_router.py b/vllm/entrypoints/serve/dev/sleep/api_router.py index fedff8da74..846a487706 100644 --- a/vllm/entrypoints/serve/dev/sleep/api_router.py +++ b/vllm/entrypoints/serve/dev/sleep/api_router.py @@ -27,6 +27,12 @@ async def sleep(raw_request: Request): return Response(status_code=200) +@router.post("/release_kv_cache_memory") +async def release_kv_cache_memory(raw_request: Request): + await engine_client(raw_request).release_kv_cache_memory() + return Response(status_code=200) + + @router.post("/wake_up") async def wake_up(raw_request: Request): tags = raw_request.query_params.getlist("tags") diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 3e399a1a81..ad9d9bf6c7 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -1079,10 +1079,17 @@ class AsyncLLM(EngineClient): if self.logger_manager is not None: self.logger_manager.record_sleep_state(1, level) - async def wake_up(self, tags: list[str] | None = None) -> None: - await self.engine_core.wake_up_async(tags) + async def release_kv_cache_memory(self) -> None: + await self.renderer.clear_mm_cache_async() + await self.engine_core.release_kv_cache_memory_async() if self.logger_manager is not None: + self.logger_manager.record_sleep_state(1, 0) + + async def wake_up(self, tags: list[str] | None = None) -> None: + fully_awake = await self.engine_core.wake_up_async(tags) + + if self.logger_manager is not None and fully_awake: self.logger_manager.record_sleep_state(0, 0) async def checkpoint_prepare(self) -> None: diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 14224f95e0..5ee2c6f8c0 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -957,12 +957,15 @@ class EngineCore: pause_future.add_done_callback(pause_complete) return future - def wake_up(self, tags: list[str] | None = None): + def wake_up(self, tags: list[str] | None = None) -> bool: """Wake up the engine from sleep. Args: tags: Tags to wake up. Use ["scheduling"] for level 0 wake up. + Returns: + Whether all executor memory is resident again (fully awake). + """ if tags is not None and "scheduling" in tags: # Remove "scheduling" from tags if there are other tags to process. @@ -973,8 +976,30 @@ class EngineCore: # Partial wakes intentionally keep the remaining allocations asleep. # Resume scheduling only once all executor memory is resident again. - if not self.model_executor.is_sleeping: + fully_awake = not self.model_executor.is_sleeping + if fully_awake: self.resume_scheduler() + return fully_awake + + def release_kv_cache_memory(self) -> None: + """Discard KV cache physical memory. Requires a completed pause + and all executor memory to be resident. Kept requests are recomputed + after wake-up. + """ + if not ( + self.is_scheduler_paused() + and not self.scheduler.has_requests() + and not self.batch_queue + ): + raise RuntimeError( + "release_kv_cache_memory() requires a completed pause first" + ) + if self.model_executor.is_sleeping: + raise RuntimeError( + "release_kv_cache_memory() requires all executor memory to be resident" + ) + self._reset_caches() + self.model_executor.discard(("kv_cache",)) def is_sleeping(self) -> bool: """Check if engine is sleeping at any level.""" diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 4ca565d56c..399af7c0b9 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -193,7 +193,10 @@ class EngineCoreClient(ABC): def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None: raise NotImplementedError - def wake_up(self, tags: list[str] | None = None) -> None: + def release_kv_cache_memory(self) -> None: + raise NotImplementedError + + def wake_up(self, tags: list[str] | None = None) -> bool: raise NotImplementedError def is_sleeping(self) -> bool: @@ -285,7 +288,10 @@ class EngineCoreClient(ABC): async def sleep_async(self, level: int = 1, mode: PauseMode = "abort") -> None: raise NotImplementedError - async def wake_up_async(self, tags: list[str] | None = None) -> None: + async def release_kv_cache_memory_async(self) -> None: + raise NotImplementedError + + async def wake_up_async(self, tags: list[str] | None = None) -> bool: raise NotImplementedError async def is_sleeping_async(self) -> bool: @@ -397,8 +403,11 @@ class InprocClient(EngineCoreClient): result = self.engine_core.sleep(level, mode) assert result is None - def wake_up(self, tags: list[str] | None = None) -> None: - self.engine_core.wake_up(tags) + def release_kv_cache_memory(self) -> None: + self.engine_core.release_kv_cache_memory() + + def wake_up(self, tags: list[str] | None = None) -> bool: + return self.engine_core.wake_up(tags) def is_sleeping(self) -> bool: return self.engine_core.is_sleeping() @@ -1021,8 +1030,11 @@ class SyncMPClient(MPClient): def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None: self.call_utility("sleep", level, mode) - def wake_up(self, tags: list[str] | None = None) -> None: - self.call_utility("wake_up", tags) + def release_kv_cache_memory(self) -> None: + self.call_utility("release_kv_cache_memory") + + def wake_up(self, tags: list[str] | None = None) -> bool: + return self.call_utility("wake_up", tags) def is_sleeping(self) -> bool: return self.call_utility("is_sleeping") @@ -1265,8 +1277,11 @@ class AsyncMPClient(MPClient): async def sleep_async(self, level: int = 1, mode: PauseMode = "abort") -> None: await self.call_utility_async("sleep", level, mode) - async def wake_up_async(self, tags: list[str] | None = None) -> None: - await self.call_utility_async("wake_up", tags) + async def release_kv_cache_memory_async(self) -> None: + await self.call_utility_async("release_kv_cache_memory") + + async def wake_up_async(self, tags: list[str] | None = None) -> bool: + return await self.call_utility_async("wake_up", tags) async def is_sleeping_async(self) -> bool: return await self.call_utility_async("is_sleeping") diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index c6c77f72e7..a2d97b5544 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -375,10 +375,17 @@ class LLMEngine: if self.logger_manager is not None: self.logger_manager.record_sleep_state(1, level) - def wake_up(self, tags: list[str] | None = None): - self.engine_core.wake_up(tags) + def release_kv_cache_memory(self) -> None: + self.renderer.clear_mm_cache() + self.engine_core.release_kv_cache_memory() if self.logger_manager is not None: + self.logger_manager.record_sleep_state(1, 0) + + def wake_up(self, tags: list[str] | None = None): + fully_awake = self.engine_core.wake_up(tags) + + if self.logger_manager is not None and fully_awake: self.logger_manager.record_sleep_state(0, 0) def is_sleeping(self) -> bool: diff --git a/vllm/v1/executor/abstract.py b/vllm/v1/executor/abstract.py index 6b07bad17c..f234e943b4 100644 --- a/vllm/v1/executor/abstract.py +++ b/vllm/v1/executor/abstract.py @@ -32,6 +32,9 @@ logger = init_logger(__name__) _R = TypeVar("_R") +# Memory-pool tags managed by sleep/wake_up/discard. +SLEEP_TAGS = frozenset(("weights", "kv_cache")) + FailureCallback = Callable[[], None] @@ -108,7 +111,6 @@ class Executor(ABC): self.speculative_config = vllm_config.speculative_config self.observability_config = vllm_config.observability_config self._init_executor() - self.is_sleeping = False self.sleeping_tags: set[str] = set() self.kv_output_aggregator: KVOutputAggregator | None = None self.ec_output_aggregator: ECOutputAggregator | None = None @@ -338,15 +340,18 @@ class Executor(ABC): """Reset the encoder cache in each worker to clear cached encoder outputs.""" self.collective_rpc("reset_encoder_cache") + @property + def is_sleeping(self) -> bool: + return bool(self.sleeping_tags) + def sleep(self, level: int = 1): - if self.is_sleeping: + if "weights" in self.sleeping_tags: logger.warning("Executor is already sleeping.") return time_before_sleep = time.perf_counter() self.collective_rpc("sleep", kwargs=dict(level=level)) time_after_sleep = time.perf_counter() - self.sleeping_tags = {"weights", "kv_cache"} - self.is_sleeping = True + self.sleeping_tags |= SLEEP_TAGS logger.info( "It took %.6f seconds to fall asleep.", time_after_sleep - time_before_sleep ) @@ -375,8 +380,23 @@ class Executor(ABC): self.sleeping_tags.remove(tag) else: self.sleeping_tags.clear() - if not self.sleeping_tags: - self.is_sleeping = False + + def discard(self, tags: tuple[str, ...]) -> None: + tags_to_discard = set(tags) - self.sleeping_tags + if not tags_to_discard: + logger.warning("Tags %s are already sleeping.", tags) + return + time_before_discard = time.perf_counter() + try: + self.collective_rpc("discard", args=(tuple(tags_to_discard),)) + finally: + self.sleeping_tags |= tags_to_discard + time_after_discard = time.perf_counter() + logger.info( + "It took %.6f seconds to discard tags %s.", + time_after_discard - time_before_discard, + tags_to_discard, + ) def reinitialize_distributed( self, reconfig_request: ReconfigureDistributedRequest diff --git a/vllm/v1/worker/cpu_worker.py b/vllm/v1/worker/cpu_worker.py index 2ee2164dc3..1badcfe6e8 100644 --- a/vllm/v1/worker/cpu_worker.py +++ b/vllm/v1/worker/cpu_worker.py @@ -198,6 +198,9 @@ class CPUWorker(Worker): logger.warning("sleep mode is not supported on CPU, ignore it.") pass + def discard(self, tags: tuple[str, ...]) -> None: + raise RuntimeError("Selective discard is not supported on CPU.") + def _should_warm_up_model(self) -> bool: # VLLM_CPU_CI_ENV always skips warmup to save CI time. if envs.VLLM_CPU_CI_ENV: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 02f599e57e..8ced3bdd43 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -304,6 +304,9 @@ class Worker(WorkerBase): self.synchronize_device() + def discard(self, tags: tuple[str, ...]) -> None: + self.sleep_mode_backend.discard(tags) + def checkpoint_prepare(self) -> None: checkpoint_prepare_distributed_state() From 48f663c37f8c77803d9281778d49ec0727abcacf Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Fri, 18 Sep 2026 07:52:01 +0800 Subject: [PATCH 068/143] [XPU] Fix incorrect context-key normalization for Qwen DFlash-based models (#56431) Signed-off-by: Yan Ma --- vllm/model_executor/models/qwen3_dflash.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index ce3884a10e..5504304c73 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -32,6 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from vllm.multimodal.inputs import NestedTensors +from vllm.platforms import current_platform from vllm.transformers_utils.config import set_default_rope_theta from vllm.transformers_utils.repo_utils import get_hf_file_bytes from vllm.v1.attention.backend import AttentionType @@ -561,6 +562,15 @@ class DFlashQwen3Model(nn.Module): # --- Grouped RMSNorm K across all layers ([L, num_ctx, nkv, hd]) --- # The weight is selected per layer by the outermost (layer) index. all_k_normed = torch.empty_like(all_k) + if current_platform.is_xpu(): + for layer_idx in range(all_k.shape[0]): + ops.rms_norm( + all_k_normed[layer_idx], + all_k[layer_idx], + self._k_norm_weights[layer_idx], + self._rms_norm_eps, + ) + return all_k_normed ops.rms_norm( all_k_normed, all_k, From 213c379fcafe5ac3bdc235a480e13140ee13c7b5 Mon Sep 17 00:00:00 2001 From: Rohan Potdar Date: Thu, 17 Sep 2026 19:06:05 -0500 Subject: [PATCH 069/143] [ROCm][Bugfix] Gate AITER MXFP8 MoE on the aiter enable flag (#57426) Signed-off-by: Rohan Potdar Co-authored-by: Claude --- .../moe/test_mxfp8_aiter_backend_selection.py | 37 +++++++++------ tests/models/quantization/test_mxfp8.py | 15 ------ .../fused_moe/experts/aiter_mxfp8_moe.py | 47 +------------------ 3 files changed, 24 insertions(+), 75 deletions(-) diff --git a/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py b/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py index 51aff4ea3b..61b26af62c 100644 --- a/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py +++ b/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py @@ -2,10 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """MXFP8 MoE backend selection for the AITER FlyDSL kernel (gfx950). -GPU-free: mocks the platform (gfx950) and the ``flydsl`` package check, then +GPU-free: mocks the platform (gfx950) and the aiter fused-MoE enable flag, then exercises the oracle so the FlyDSL backend is auto-picked when usable (including under expert parallelism, since apply() forwards the expert_map as aiter's -expert_mask) and skipped (native fallback) when the device/package is missing. +expert_mask) and skipped (native fallback) when the device is unsupported or the +aiter runtime is disabled. """ import dataclasses @@ -48,6 +49,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: kMxfp8Static, ) +from vllm._aiter_ops import rocm_aiter_ops # noqa: E402 # isort: skip + _AITER_MOD = "vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe" @@ -82,8 +85,8 @@ def _gfx950(): ) -def _flydsl_installed(present: bool): - return patch(f"{_AITER_MOD}.is_aiter_mxfp8_moe_available", return_value=present) +def _aiter_moe_enabled(present: bool): + return patch.object(rocm_aiter_ops, "is_fused_moe_enabled", return_value=present) def test_aiter_mxfp8_registered(): @@ -109,13 +112,18 @@ def test_ep_supported(ep_size): @pytest.mark.parametrize( "present,ep_size,supported,reason_substr", [ - (True, 1, True, None), # gfx950 + flydsl + TP -> selectable - (True, 2, True, None), # gfx950 + flydsl + EP -> selectable (expert_mask) - (False, 1, False, "flydsl package"), # package missing -> clear reason + (True, 1, True, None), # gfx950 + aiter MoE + TP -> selectable + (True, 2, True, None), # gfx950 + aiter MoE + EP -> selectable (expert_mask) + ( + False, + 1, + False, + "does not support current device", + ), # disabled -> not selected ], ) def test_is_supported_config(present, ep_size, supported, reason_substr): - with _gfx950(), _flydsl_installed(present): + with _gfx950(), _aiter_moe_enabled(present): ok, reason = AiterMxfp8Experts.is_supported_config( AiterMxfp8Experts, _config(ep_size), @@ -130,8 +138,8 @@ def test_is_supported_config(present, ep_size, supported, reason_substr): def test_explicit_moe_backend_aiter(): """--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear - ValueError when the flydsl package is missing.""" - with _gfx950(), _flydsl_installed(True): + ValueError when the aiter MoE runtime is disabled.""" + with _gfx950(), _aiter_moe_enabled(True): assert ( _select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1)) is AiterMxfp8Experts @@ -142,18 +150,17 @@ def test_explicit_moe_backend_aiter(): ) with ( _gfx950(), - _flydsl_installed(False), - pytest.raises(ValueError, match="flydsl package"), + _aiter_moe_enabled(False), + pytest.raises(ValueError, match="does not support current device"), ): _select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1)) def test_gfx950_picks_aiter(): - """Auto-select on real ROCm hardware with flydsl usable -> FlyDSL wins.""" - # NOTE: Fp8MoeBackend.AITER_MXFP8 does not require VLLM_ROCM_USE_AITER=1 + """Auto-select on real ROCm hardware with aiter MoE enabled -> FlyDSL wins.""" with ( patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=True), - _flydsl_installed(True), + _aiter_moe_enabled(True), ): backend, experts_cls = select_mxfp8_moe_backend(_config()) assert backend is Fp8MoeBackend.AITER_MXFP8 diff --git a/tests/models/quantization/test_mxfp8.py b/tests/models/quantization/test_mxfp8.py index c12a72a09c..0463468eae 100644 --- a/tests/models/quantization/test_mxfp8.py +++ b/tests/models/quantization/test_mxfp8.py @@ -111,11 +111,6 @@ def test_mxfp8_aiter_requires_swigluoai_activation( "_supports_current_device", staticmethod(lambda: True), ) - monkeypatch.setattr( - aiter_mxfp8_moe, - "is_aiter_mxfp8_moe_available", - lambda: True, - ) config = FusedMoEConfig( num_experts=8, @@ -164,11 +159,6 @@ def test_mxfp8_aiter_requires_swigluoai_params( "_supports_current_device", staticmethod(lambda: True), ) - monkeypatch.setattr( - aiter_mxfp8_moe, - "is_aiter_mxfp8_moe_available", - lambda: True, - ) config = FusedMoEConfig( num_experts=8, @@ -218,11 +208,6 @@ def test_mxfp8_aiter_accepts_swigluoai_params( "_supports_current_device", staticmethod(lambda: True), ) - monkeypatch.setattr( - aiter_mxfp8_moe, - "is_aiter_mxfp8_moe_available", - lambda: True, - ) config = FusedMoEConfig( num_experts=8, diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py index 93a4088fc4..c15a1a13d4 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py @@ -11,6 +11,7 @@ import math import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm._aiter_ops import rocm_aiter_ops from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import ( @@ -24,41 +25,6 @@ _AITER_SWIGLU_ALPHA = 1.702 _AITER_SWIGLU_BETA = 1.0 -def is_aiter_mxfp8_moe_available() -> bool: - """True when the FlyDSL MXFP8 MoE can run here: gfx950, the ``flydsl`` - package is importable, AND the installed aiter carries the mxfp8 FlyDSL - 2-stage support from ROCm/aiter#3811. - - ``flydsl`` and ``aiter`` are separate packages, so ``is_flydsl_available()`` - (flydsl pkg + arch) is necessary but not sufficient: an older aiter without - #3811 still ships the flydsl pkg and the ``aiter.ops.flydsl`` module but a - broken/missing ``per_1x32 + fp8`` 2-stage path. Without this extra gate a - nightly lacking #3811 would wrongly select FlyDSL instead of falling back to - the native Triton dot_scaled path. #3811 added no probe-able public symbol, - so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails - closed (returns False -> triton dot_scaled), which is always safe.""" - if not (current_platform.is_rocm() and current_platform.supports_mx()): - return False - try: - import os - - import aiter - from aiter.ops.flydsl.utils import is_flydsl_available - - if not is_flydsl_available(): - return False - return os.path.exists( - os.path.join( - os.path.dirname(aiter.__file__), - "configs", - "model_configs", - "minimax_m3_mxfp8_tuned_fmoe.csv", - ) - ) - except Exception: - return False - - class AiterMxfp8Experts(Mxfp8TritonExpertsBase): """MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950).""" @@ -79,10 +45,7 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase): @staticmethod def _supports_current_device() -> bool: - # Device capability only (gfx950 / MX-capable ROCm). The flydsl package - # check lives in is_supported_config so a missing package is reported - # distinctly from an unsupported device. - return current_platform.is_rocm() and current_platform.supports_mx() + return current_platform.supports_mx() and rocm_aiter_ops.is_fused_moe_enabled() @staticmethod def _supports_parallel_config(moe_parallel_config) -> bool: @@ -95,12 +58,6 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase): is_supported, reason = super().is_supported_config( cls, moe_config, weight_key, activation_key, activation_format ) - # _supports_current_device() only gates on the device; surface a clear - # reason when the device is fine but the flydsl package is missing. - if is_supported and not is_aiter_mxfp8_moe_available(): - return False, ( - "kernel requires the aiter flydsl package, which is not installed" - ) if ( is_supported and moe_config.activation != MoEActivation.SWIGLUOAI_UNINTERLEAVE From 2c88fb131c7ae0be01907cd8c276911db5e7aad4 Mon Sep 17 00:00:00 2001 From: Matt Mastracci Date: Thu, 17 Sep 2026 18:24:58 -0600 Subject: [PATCH 070/143] [Bugfix] DiffusionGemma: hand out stashed logprobs only on the committing step (#57414) --- vllm/model_executor/models/diffusion_gemma.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index bd1d14b6f6..af0cb35168 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -1312,7 +1312,6 @@ class DiffusionSampler: is_committing = states.is_encoder_phase[decode_slots].clone() slots_np = input_batch.idx_mapping_np[:num_reqs] - is_decode_np = per_req_nlogits_np > 0 max_num_logprobs = self.sampling_states.max_num_logprobs(slots_np) # Sample over the [num_decode * CL, vocab] logits. The fp32 pipeline in @@ -1396,13 +1395,16 @@ class DiffusionSampler: # stashed logprobs and attach to SamplerOutput. logprobs_tensors = None if max_num_logprobs >= 0 and is_committing.any() and self._pending_logprobs: + committing_slots = set( + decode_slots_np[is_committing.cpu().numpy()].tolist() + ) parts_ids, parts_lp, parts_ranks = [], [], [] cu_gen: list[int] = [] flat_offset = 0 for i in range(num_reqs): cu_gen.append(flat_offset) slot = int(slots_np[i]) - if is_decode_np[i] and slot in self._pending_logprobs: + if slot in committing_slots and slot in self._pending_logprobs: lp = self._pending_logprobs.pop(slot) parts_ids.append(lp.logprob_token_ids) parts_lp.append(lp.logprobs) From 64563d0ec4b761745d5babbc5750b263f0783926 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Thu, 17 Sep 2026 19:00:10 -0700 Subject: [PATCH 071/143] [Bugfix][DSv4.1] Preserve NaN-scored candidate block indices (#57454) Signed-off-by: Woosuk Kwon Co-authored-by: Codex --- .../layers/test_mla_short_prefill_indexer.py | 22 ++++++++++++++++++- .../kernels/attention/dsa/candidate_blocks.py | 3 ++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 41a06d0546..882b745ece 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -365,6 +365,26 @@ def test_select_candidate_blocks_tolerates_empty_rows(): assert (out[0] >= 0).all() and (out[2, :2] >= 0).all() +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("score", [0.0, float("nan")]) +@pytest.mark.parametrize("start", [0, 3]) +def test_candidate_selection_keeps_valid_blocks_with_nan_scores(score, start): + """NaN warmup scores must not turn real blocks into repeated sparse padding.""" + logits = torch.full((3, 64), score, device="cuda") + starts = torch.full((3,), start, device="cuda", dtype=torch.int32) + lengths = torch.tensor([0, 17, 35], device="cuda", dtype=torch.int32) + out = torch.empty(3, 8, device="cuda", dtype=torch.int32) + + sparse_indexer._select_candidate_blocks(logits, starts, starts + lengths, 8, 8, out) + + for row, count in enumerate([0, 3, 5]): + valid = out[row][out[row] >= 0].sort().values + torch.testing.assert_close( + valid, torch.arange(count, device="cuda", dtype=torch.int32) + ) + assert (out[row] == -1).sum().item() == 8 - count + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize( "width,block_size,k,decode", @@ -413,7 +433,7 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding( top = reduced.topk(min(k, nblocks), dim=-1) expected = torch.full((rows, k), -1, device="cuda", dtype=torch.int32) expected[:, : top.indices.shape[1]] = torch.where( - top.values > -torch.inf, top.indices, -1 + top.values != -torch.inf, top.indices, -1 ).int() actual = torch.empty(rows, k * 2, device="cuda", dtype=torch.int32)[:, ::2] select_candidate_blocks(logits, starts, ends, k, block_size, actual, repeat) diff --git a/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py b/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py index efc7c418ca..b9cdc923b1 100644 --- a/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py +++ b/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py @@ -66,9 +66,10 @@ def _store_candidates_kernel( cols = tl.program_id(1) * TILE + tl.arange(0, TILE) value = tl.load(values + row * k + cols, cols < k, other=-float("inf")) index = tl.load(indices + row * k + cols, cols < k, other=-1) + # NaN scores can occur during warmup; only -inf denotes padding. tl.store( output + row * out_stride_row + cols * out_stride_col, - tl.where(value > -float("inf"), index, -1), + tl.where(value != -float("inf"), index, -1), cols < OUT_K, ) From 2bbdfcfcfea79db5a3a4a32a573e0998fd6a58eb Mon Sep 17 00:00:00 2001 From: Chauncey Date: Fri, 18 Sep 2026 10:31:35 +0800 Subject: [PATCH 072/143] [Perf][GLM5.3-Flash] Use cooperative top-k for small GLM decode batches (#57327) Signed-off-by: chaunceyjiang Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../test_sparse_indexer_topk_dispatch.py | 41 +++++++++++++++++++ vllm/models/glm5next/nvidia/sparse_indexer.py | 23 ++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/models/glm5next/test_sparse_indexer_topk_dispatch.py diff --git a/tests/models/glm5next/test_sparse_indexer_topk_dispatch.py b/tests/models/glm5next/test_sparse_indexer_topk_dispatch.py new file mode 100644 index 0000000000..83dfdc9f6f --- /dev/null +++ b/tests/models/glm5next/test_sparse_indexer_topk_dispatch.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Focused unit tests for the GLM decode top-k dispatch.""" + +import pytest +import torch + +from vllm.models.glm5next.nvidia.sparse_indexer import _use_cooperative_topk +from vllm.platforms import current_platform + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda(), reason="CUDA-only dispatch" +) + + +@pytest.mark.parametrize("num_rows", [1, 64], ids=["1row", "64rows"]) +@torch.inference_mode() +def test_cooperative_topk_is_selected_for_supported_batch(num_rows: int): + logits = torch.empty(num_rows, 8192, dtype=torch.float32, device="cuda") + assert _use_cooperative_topk(logits, 512, num_rows) + + +@pytest.mark.parametrize( + ("num_rows", "select_k", "stride_padding"), + [ + (65, 512, 0), + (1, 256, 0), + (1, 512, 1), + ], + ids=["too_many_rows", "unsupported_k", "unaligned_stride"], +) +@torch.inference_mode() +def test_cooperative_topk_falls_back_on_unsupported_batch( + num_rows: int, select_k: int, stride_padding: int +): + logits = torch.empty( + num_rows, 8192 + stride_padding, dtype=torch.float32, device="cuda" + ) + if stride_padding: + logits = logits[:, :8192] + assert not _use_cooperative_topk(logits, select_k, num_rows) diff --git a/vllm/models/glm5next/nvidia/sparse_indexer.py b/vllm/models/glm5next/nvidia/sparse_indexer.py index 1fc42c9e41..9dab659d4c 100644 --- a/vllm/models/glm5next/nvidia/sparse_indexer.py +++ b/vllm/models/glm5next/nvidia/sparse_indexer.py @@ -38,6 +38,22 @@ if current_platform.is_cuda_alike(): logger = init_logger(__name__) + +def _use_cooperative_topk( + logits: torch.Tensor, + select_k: int, + num_rows: int, +) -> bool: + return ( + current_platform.is_cuda() + and select_k in (512, 1024, 2048) + and num_rows <= 64 + and logits.stride(0) % 4 == 0 + and current_platform.has_device_capability(90) + and not current_platform.is_device_capability_family(120) + ) + + # kpool write helper: form pools from the current token batch and compress them # into the index K cache via the fused Triton kernel. @@ -573,7 +589,12 @@ def sparse_attn_indexer_kpool( (topk_workspace,) = workspace_manager.get_simultaneous( ((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8), ) - torch.ops._C.persistent_topk( + topk_op = ( + torch.ops._C.cooperative_topk + if _use_cooperative_topk(logits, select_k, num_rows) + else torch.ops._C.persistent_topk + ) + topk_op( logits, seq_lens, topk_dst, From 7b942936276d59cc1912185a442b97dea8d2386c Mon Sep 17 00:00:00 2001 From: Ren Yuzhou <54501155+yuzhouo7@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:35:23 +0800 Subject: [PATCH 073/143] [Bugfix][ROCm][KV Offload] Use private pinned tensors for CPU KV offload (#57160) Signed-off-by: Yuzhou Ren --- tests/v1/kv_offload/test_factory.py | 49 +++++++++++++++++++---------- vllm/v1/kv_offload/cpu/spec.py | 2 +- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tests/v1/kv_offload/test_factory.py b/tests/v1/kv_offload/test_factory.py index 16539677c4..01ef756cae 100644 --- a/tests/v1/kv_offload/test_factory.py +++ b/tests/v1/kv_offload/test_factory.py @@ -275,11 +275,12 @@ def test_tiering_spec_create_worker_folds_device_index_for_sharded_layout(monkey @pytest.mark.parametrize("world_size", [2, 4, 8]) def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: int): - # On shared-region (CUDA-alike) platforms the default spec now honors + # On CUDA the default spec honors # replicated layout: a single MLA copy (num_copies=1), matching tiering. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, @@ -295,19 +296,19 @@ def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: in assert spec.num_chunks == 8 +@pytest.mark.parametrize("rocm", [False, True], ids=["non-cuda-alike", "rocm"]) @pytest.mark.parametrize("world_size", [2, 4, 8]) def test_cpu_spec_replicated_disabled_without_shared_region( - monkeypatch, world_size: int + monkeypatch, world_size: int, rocm: bool ): - # Data-loss guard: non-CUDA-alike platforms keep a per-rank private pinned + # Data-loss guard: platforms without shared regions keep a private pinned # tensor (no shared medium), so replicated layout MUST stay off. Otherwise # the rank-0 writer gate would ack rank>0 stores without writing, leaving # those private buffers empty and corrupting subsequent loads. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module - monkeypatch.setattr( - cpu_spec_module.current_platform, "is_cuda_alike", lambda: False - ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: rocm) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * world_size * 2, @@ -324,17 +325,26 @@ def test_cpu_spec_replicated_disabled_without_shared_region( @pytest.mark.parametrize("config_replicated", [True, False]) -@pytest.mark.parametrize("cuda_alike", [True, False]) +@pytest.mark.parametrize( + ("cuda_alike", "rocm", "shared_region"), + [(True, False, True), (True, True, False), (False, False, False)], + ids=["cuda", "rocm", "non-cuda-alike"], +) def test_cpu_spec_replicated_layout_truth_matrix( - monkeypatch, cuda_alike: bool, config_replicated: bool + monkeypatch, + cuda_alike: bool, + rocm: bool, + shared_region: bool, + config_replicated: bool, ): # replicated_layout is enabled iff the config gate passes AND the deployment - # actually allocates on the shared region (CUDA-alike). + # actually allocates on the shared region. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr( cpu_spec_module.current_platform, "is_cuda_alike", lambda: cuda_alike ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, @@ -344,10 +354,10 @@ def test_cpu_spec_replicated_layout_truth_matrix( ) assert isinstance(spec, CPUOffloadingSpec) - assert spec.replicated_layout is (cuda_alike and config_replicated) + assert spec.replicated_layout is (shared_region and config_replicated) -def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): +def test_cpu_spec_create_worker_uses_mmap_on_cuda(monkeypatch): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT @@ -371,6 +381,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): return MagicMock() monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", fake_worker_ctor) monkeypatch.setattr( @@ -386,7 +397,10 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): assert worker_calls[0]["mmap_region"] is region -def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch): +@pytest.mark.parametrize("rocm", [False, True], ids=["non-cuda-alike", "rocm"]) +def test_cpu_spec_create_worker_uses_tensor_path_without_shared_region( + monkeypatch, rocm +): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module spec = _create_spec(worker_kv_bytes_per_block=4096, world_size=4) @@ -403,15 +417,14 @@ def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch): worker_calls.append(kwargs) return MagicMock() - monkeypatch.setattr( - cpu_spec_module.current_platform, "is_cuda_alike", lambda: False - ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: rocm) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", fake_worker_ctor) spec.create_worker(MagicMock()) - # Non-CUDA-alike platforms keep the per-rank pinned-tensor path. + # ROCm and non-CUDA-alike platforms use the per-rank pinned-tensor path. assert region_calls == [] assert worker_calls[0]["mmap_region"] is None @@ -420,7 +433,7 @@ def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module # worker_kv_bytes_per_block=0 yields num_chunks=0; a zero-byte region cannot - # be mmap'd, so even on CUDA-alike this must fall back to the tensor path. + # be mmap'd, so even on CUDA this must fall back to the tensor path. spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4) assert isinstance(spec, CPUOffloadingSpec) assert spec.num_chunks == 0 @@ -429,6 +442,7 @@ def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch): worker_calls: list[dict[str, Any]] = [] monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) monkeypatch.setattr( cpu_spec_module, "SharedOffloadRegion", @@ -461,6 +475,7 @@ def test_cpu_spec_create_worker_rank_assignment( import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 00b36e863a..fddf566085 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -162,7 +162,7 @@ class CPUOffloadingSpec(OffloadingSpec): def _uses_shared_region(self) -> bool: """Whether the worker CPU buffer is the shared mmap region (vs a private per-rank tensor); replicated-layout dedup is gated on this being True.""" - return current_platform.is_cuda_alike() + return current_platform.is_cuda_alike() and not current_platform.is_rocm() def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker: mmap_region: SharedOffloadRegion | None = None From 70df48dc3d01fe1bb3206f7d8a832c7e8de348e8 Mon Sep 17 00:00:00 2001 From: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Date: Fri, 18 Sep 2026 04:42:54 +0200 Subject: [PATCH 074/143] [Bugfix][KV Cache][GLM-5.3-Flash] Disable slot mapping kernel for the kpool tail buffer (#57317) Signed-off-by: simondanielsson --- .../attention/test_kpool_tail_slot_mapping.py | 24 +++++ .../test_gpu_kpool_tail_slot_mapping.py | 98 +++++++++++++++++++ vllm/v1/kv_cache_interface.py | 4 + 3 files changed, 126 insertions(+) create mode 100644 tests/v1/worker/test_gpu_kpool_tail_slot_mapping.py diff --git a/tests/v1/attention/test_kpool_tail_slot_mapping.py b/tests/v1/attention/test_kpool_tail_slot_mapping.py index 0966f9b20a..cb39e54037 100644 --- a/tests/v1/attention/test_kpool_tail_slot_mapping.py +++ b/tests/v1/attention/test_kpool_tail_slot_mapping.py @@ -32,6 +32,7 @@ from vllm.v1.attention.backends.mla.indexer import ( ) from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides from vllm.v1.kv_cache_layout import KVCacheLayout +from vllm.v1.worker.block_table import get_block_table_width KPOOL = 4 @@ -55,6 +56,29 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic(): assert content_stride == 1 +def test_tail_spec_opts_out_of_generic_slot_mapping(): + """The tail row is one block wide (padded to the block-table alignment), so + the generic kernel's ``pos // kpool`` column index runs off the end of the + allocation for long prompts. The spec must opt out of it entirely.""" + spec = KpoolTailSpec( + block_size=KPOOL, + num_kv_heads=2, + head_size=128, + head_size_v=0, + dtype=torch.bfloat16, + sliding_window=KPOOL, + ) + max_len = 1 << 20 + width = get_block_table_width( + spec.max_num_blocks_per_req(None, max_len), + spec.block_size, + token_alignment=spec.block_table_token_alignment, + ) + + assert width * KPOOL < max_len + assert spec.uses_slot_mapping is False + + def make_tail_block_table(own_blocks, width=64): """Tail-group block table as BlockTables produces it: column 0 holds the request's single KpoolTailManager block, the remaining columns are never diff --git a/tests/v1/worker/test_gpu_kpool_tail_slot_mapping.py b/tests/v1/worker/test_gpu_kpool_tail_slot_mapping.py new file mode 100644 index 0000000000..8a7445f7d4 --- /dev/null +++ b/tests/v1/worker/test_gpu_kpool_tail_slot_mapping.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GPU regression test for the kpool tail group's slot mapping. + +The kpool tail cache holds one circular block per request, so its block table +row is orders of magnitude shorter than the sequence it serves. +`_compute_slot_mappings_kernel` indexes block tables by +`position // kernel_block_size` with an effectively unmasked load, so +this leads to a read of neighboring requests' rows (and for the later requests, outside +the buffer entirely). +""" + +import pytest +import torch + +from vllm.platforms import current_platform +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.kv_cache_interface import KpoolTailSpec +from vllm.v1.worker.block_table import get_block_table_width +from vllm.v1.worker.gpu.block_table import BlockTables + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda_alike(), + reason="requires CUDA or ROCm", +) + +KPOOL = 4 +MAX_MODEL_LEN = 1 << 20 +MLA_BLOCK_SIZE = 640 +MAX_NUM_REQS = 4 +POISON_BLOCK_ID = 1 << 20 + + +def make_tail_spec(): + return KpoolTailSpec( + block_size=KPOOL, + num_kv_heads=2, + head_size=128, + head_size_v=0, + dtype=torch.bfloat16, + sliding_window=KPOOL, + ) + + +def test_kpool_tail_group_never_position_indexes_its_block_table(): + device = torch.device("cuda") + spec = make_tail_spec() + tail_width = get_block_table_width( + spec.max_num_blocks_per_req(None, MAX_MODEL_LEN), + spec.block_size, + token_alignment=spec.block_table_token_alignment, + ) + mla_width = get_block_table_width(MAX_MODEL_LEN // MLA_BLOCK_SIZE, MLA_BLOCK_SIZE) + + # Positions chosen against the tail row: `in_row` still lands inside + # request 0's own row, `next_row` lands on request 1's poisoned row, and + # `past_table` is past the whole [MAX_NUM_REQS, tail_width] allocation. + in_row = KPOOL - 1 + next_row = tail_width * KPOOL + past_table = MAX_NUM_REQS * tail_width * KPOOL + KPOOL + assert past_table < MAX_MODEL_LEN + positions = [0, in_row, next_row, past_table] + + block_tables = BlockTables( + block_sizes=[MLA_BLOCK_SIZE, spec.block_size], + max_num_reqs=MAX_NUM_REQS, + max_num_batched_tokens=len(positions), + max_num_blocks_per_group=[mla_width, tail_width], + device=device, + kernel_block_sizes=[MLA_BLOCK_SIZE, spec.block_size], + slot_mapping_enabled=[True, spec.uses_slot_mapping], + ) + mla_blocks = list(range(1, mla_width + 1)) + block_tables.append_block_ids(0, (mla_blocks, [7]), overwrite=True) + block_tables.append_block_ids(1, ([2], [POISON_BLOCK_ID]), overwrite=True) + block_tables.apply_staged_writes() + + idx_mapping = torch.zeros(1, dtype=torch.int32, device=device) + query_start_loc = torch.tensor( + [0, len(positions)], dtype=torch.int32, device=device + ) + slot_mappings = block_tables.compute_slot_mappings( + idx_mapping, + query_start_loc, + torch.tensor(positions, dtype=torch.int64, device=device), + num_tokens_padded=len(positions), + ) + torch.accelerator.synchronize() + + tail_slots = slot_mappings[1].tolist() + assert tail_slots == [PAD_SLOT_ID] * len(positions) + assert POISON_BLOCK_ID * KPOOL not in tail_slots + + mla_slots = slot_mappings[0].tolist() + assert mla_slots == [ + mla_blocks[p // MLA_BLOCK_SIZE] * MLA_BLOCK_SIZE + p % MLA_BLOCK_SIZE + for p in positions + ] diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 17771d9ae6..6595be14d2 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -987,6 +987,10 @@ class KpoolTailSpec(SlidingWindowSpec): def prefix_cacheable(self) -> bool: return False + @property + def uses_slot_mapping(self) -> bool: + return False + @dataclass(frozen=True) class MambaSpec(KVCacheSpec): From 2909ad8fa472c04f7db8330f0e751ff43ae76965 Mon Sep 17 00:00:00 2001 From: Jason <31175216+thegoldenflow@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:07:25 -0400 Subject: [PATCH 075/143] [Bugfix] Attach request-level tools to existing system message in DeepSeek V4 Python renderer (#51856) Signed-off-by: Jason Yao Co-authored-by: Chauncey --- .../fixtures/deepseek_v4/test_output_1.txt | 4 +- tests/tokenizers_/test_deepseek_v4.py | 143 ++++++++++++++++++ vllm/tokenizers/deepseek_v4.py | 14 +- 3 files changed, 157 insertions(+), 4 deletions(-) diff --git a/tests/tokenizers_/fixtures/deepseek_v4/test_output_1.txt b/tests/tokenizers_/fixtures/deepseek_v4/test_output_1.txt index dbd823476c..7e3c9bd5a3 100644 --- a/tests/tokenizers_/fixtures/deepseek_v4/test_output_1.txt +++ b/tests/tokenizers_/fixtures/deepseek_v4/test_output_1.txt @@ -1,4 +1,4 @@ -<|begin▁of▁sentence|> +<|begin▁of▁sentence|>You are a helpful assistant. ## Tools @@ -26,7 +26,7 @@ Otherwise, output directly after with tool calls or final response. {"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}} You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. -You are a helpful assistant.<|User|>What's the weather in Beijing?<|Assistant|>The user wants to know the weather in Beijing. I should use the get_weather tool. +<|User|>What's the weather in Beijing?<|Assistant|>The user wants to know the weather in Beijing. I should use the get_weather tool. <|DSML|tool_calls> <|DSML|invoke name="get_weather"> diff --git a/tests/tokenizers_/test_deepseek_v4.py b/tests/tokenizers_/test_deepseek_v4.py index a55aa57666..f7bc5909ba 100644 --- a/tests/tokenizers_/test_deepseek_v4.py +++ b/tests/tokenizers_/test_deepseek_v4.py @@ -485,3 +485,146 @@ def test_deepseek_v4_image_sentinel_ids_match_tokenizer(): ] ) assert image_sentinel_mask(ids).tolist() == [False, True, True, False, False] + + +def _request_tools(): + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + + +def _encode_reference(messages): + from vllm.tokenizers.deepseek_v4_encoding import encode_messages + + return encode_messages(messages, thinking_mode="thinking", reasoning_effort="low") + + +def test_deepseek_v4_attaches_request_tools_to_existing_system_message(): + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + + prompt = _tokenizer().apply_chat_template( + messages, + tools=_request_tools(), + tokenize=False, + thinking=True, + reasoning_effort="low", + ) + + expected = _encode_reference( + [{**messages[0], "tools": _request_tools()}, messages[1]] + ) + assert prompt == expected + assert prompt.startswith("<|begin▁of▁sentence|>You are helpful.\n\n## Tools") + assert prompt.count("## Tools") == 1 + + +def test_deepseek_v4_synthetic_system_only_when_no_system_message(): + """Without a system message, request tools still get a synthetic leading + system entry.""" + messages = [{"role": "user", "content": "Weather in Paris?"}] + + prompt = _tokenizer().apply_chat_template( + messages, + tools=_request_tools(), + tokenize=False, + thinking=True, + reasoning_effort="low", + ) + + expected = _encode_reference( + [{"role": "system", "tools": _request_tools()}, messages[0]] + ) + assert prompt == expected + assert prompt.startswith("<|begin▁of▁sentence|>\n\n## Tools") + + +def test_deepseek_v4_request_tools_override_system_message_tools(): + message_tools = [ + { + "type": "function", + "function": { + "name": "message_level_tool", + "description": "Should not be rendered", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + system_message = { + "role": "system", + "content": "You are helpful.", + "tools": message_tools, + } + user_message = {"role": "user", "content": "Weather in Paris?"} + + prompt = _tokenizer().apply_chat_template( + [system_message, user_message], + tools=_request_tools(), + tokenize=False, + thinking=True, + reasoning_effort="low", + ) + + expected = _encode_reference( + [{**system_message, "tools": _request_tools()}, user_message] + ) + assert prompt == expected + assert '"name": "get_weather"' in prompt + assert "message_level_tool" not in prompt + + +def test_deepseek_v4_attaches_request_tools_to_first_system_message_only(): + """Only the first system message, at any index, gets the request tools.""" + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "system", "content": "First system."}, + {"role": "system", "content": "Second system."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + + prompt = _tokenizer().apply_chat_template( + messages, + tools=_request_tools(), + tokenize=False, + thinking=True, + reasoning_effort="low", + ) + + expected = _encode_reference( + [ + messages[0], + {**messages[1], "tools": _request_tools()}, + messages[2], + messages[3], + ] + ) + assert prompt == expected + assert "First system.\n\n## Tools" in prompt + assert prompt.count("## Tools") == 1 + + +def test_deepseek_v4_request_tools_do_not_mutate_caller_messages(): + import copy + + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + snapshot = copy.deepcopy(messages) + + _tokenizer().apply_chat_template(messages, tools=_request_tools(), tokenize=False) + + assert messages == snapshot diff --git a/vllm/tokenizers/deepseek_v4.py b/vllm/tokenizers/deepseek_v4.py index a5d5020e94..532ca0df7c 100644 --- a/vllm/tokenizers/deepseek_v4.py +++ b/vllm/tokenizers/deepseek_v4.py @@ -35,8 +35,18 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: conversation = kwargs.get("conversation", messages) messages = conversation.copy() if tools is not None and len(tools) > 0: - messages.insert(0, {"role": "system"}) - messages[0]["tools"] = tools # type: ignore[typeddict-unknown-key] + # Match the Rust renderer: request tools attach to the first + # system message; synthesize one only when none exists. + system_idx = next( + (i for i, m in enumerate(messages) if m.get("role") == "system"), + None, + ) + if system_idx is None: + messages.insert(0, {"role": "system"}) + system_idx = 0 + else: + messages[system_idx] = copy.copy(messages[system_idx]) + messages[system_idx]["tools"] = tools # type: ignore[typeddict-unknown-key] reasoning_effort = kwargs.get("reasoning_effort") if not isinstance(reasoning_effort, str): From bf13ecc2f7c4d69b59854f7cef37547be58805e5 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 17 Sep 2026 20:14:36 -0700 Subject: [PATCH 076/143] [Perf][MRV2] Share token-to-request mappings across KV cache groups (#57102) Signed-off-by: Juntian Liu Co-authored-by: Woosuk Kwon --- vllm/v1/worker/gpu/attn_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index eccfdbabcd..89f54d4d11 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -426,6 +426,7 @@ def build_attn_metadata( seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound[:num_reqs] attn_metadata: dict[str, Any] = {} + token_to_req_indices: torch.Tensor | None = None num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) for i in range(num_kv_cache_groups): if not attn_groups[i]: @@ -472,6 +473,7 @@ def build_attn_metadata( mm_req_doc_ranges=mm_req_doc_ranges, rswa_prefix_lens=rswa_prefix_lens, req_idx=req_idx, + _token_to_req_indices_cache=token_to_req_indices, **common_attn_metadata_extra_kwargs, ) @@ -497,6 +499,7 @@ def build_attn_metadata( ) for layer_name in attn_group.layer_names: attn_metadata[layer_name] = metadata + token_to_req_indices = common_attn_metadata._token_to_req_indices_cache return attn_metadata From d12c2768530a4da8d7c7d4a4f5d6adfa351675cd Mon Sep 17 00:00:00 2001 From: Mustafa YILDIRIM <2341474+mustafayildirim@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:36:55 -0700 Subject: [PATCH 077/143] [Bugfix][ROCm] Alias SparseAttnIndexerKpool.forward_cuda to forward_native (GLM-5.3-Flash boot crash) (#57425) Signed-off-by: Mustafa YILDIRIM Co-authored-by: Hermes Agent Co-authored-by: Tyler Michael Smith --- vllm/models/glm5next/amd/sparse_indexer.py | 25 +++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/vllm/models/glm5next/amd/sparse_indexer.py b/vllm/models/glm5next/amd/sparse_indexer.py index f47c9164bc..c9b68e973f 100644 --- a/vllm/models/glm5next/amd/sparse_indexer.py +++ b/vllm/models/glm5next/amd/sparse_indexer.py @@ -665,7 +665,7 @@ class SparseAttnIndexerKpool(CustomOp): self.skip_k_cache_insert = skip_k_cache_insert self.use_fp4_cache = use_fp4_cache - def forward_native( + def forward_hip( self, hidden_states: torch.Tensor, q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor], @@ -727,3 +727,26 @@ class SparseAttnIndexerKpool(CustomOp): self.tail_cache.kv_cache if self.tail_cache is not None else None, self.tail_cache.prefix if self.tail_cache is not None else None, ) + + def forward_native( + self, + hidden_states: torch.Tensor, + q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor], + k: torch.Tensor, + weights: torch.Tensor, + *, + gate_score: torch.Tensor | None = None, + compress_ape: torch.Tensor | None = None, + index_kpool: int = 1, + positions: torch.Tensor | None = None, + ) -> torch.Tensor: + return self.forward_hip( + hidden_states, + q_quant, + k, + weights, + gate_score=gate_score, + compress_ape=compress_ape, + index_kpool=index_kpool, + positions=positions, + ) From dee37d89115db4c94a820a79a78a7828e141c910 Mon Sep 17 00:00:00 2001 From: lazymio Date: Fri, 18 Sep 2026 12:32:39 +0800 Subject: [PATCH 078/143] [Frontend] Add `--tool-strict-level` for server-side control for structural tag activation (#56268) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Bugen Zhao Signed-off-by: lazymio Signed-off-by: Bugen Zhao --- docs/features/tool_calling.md | 40 ++- rust/src/chat/src/backend/hf.rs | 2 + rust/src/chat/src/backend/mod.rs | 3 +- rust/src/chat/src/lib.rs | 19 +- rust/src/chat/src/output/default/mod.rs | 15 +- .../chat/src/output/default/structural_tag.rs | 220 +++++++++++++-- rust/src/chat/src/parser/mod.rs | 60 +++++ rust/src/chat/tests/chat.rs | 1 + rust/src/chat/tests/roundtrip.rs | 1 + rust/src/cmd/src/cli.rs | 16 ++ rust/src/cmd/src/cli/tests.rs | 6 + .../examples/external_engine_openai_qwen.rs | 2 + rust/src/server/src/config.rs | 3 + rust/src/server/src/grpc/tests.rs | 2 + rust/src/server/src/lib.rs | 3 +- rust/src/server/src/render.rs | 14 +- .../server/src/routes/http_client_tests.rs | 1 + rust/src/server/src/routes/tests.rs | 1 + .../test_structural_tag_registry.py | 254 +++++++++++++++++- vllm/entrypoints/anthropic/serving.py | 2 + vllm/entrypoints/cohere/serving.py | 2 + vllm/entrypoints/generate/api_router.py | 4 + .../launchers/api_server/app_state.py | 2 + vllm/entrypoints/launchers/cli_args.py | 8 + .../entrypoints/launchers/render/app_state.py | 2 + .../openai/chat_completion/serving.py | 2 + vllm/entrypoints/openai/responses/serving.py | 2 + vllm/parser/abstract_parser.py | 4 + vllm/parser/parser_manager.py | 10 + vllm/renderers/online_derenderer.py | 2 + vllm/renderers/online_renderer.py | 2 + vllm/tool_parsers/abstract_tool_parser.py | 3 + vllm/tool_parsers/hy_v4_tool_parser.py | 4 + vllm/tool_parsers/structural_tag_registry.py | 59 +++- vllm/tool_parsers/tool_strict_level.py | 32 +++ 35 files changed, 742 insertions(+), 61 deletions(-) create mode 100644 vllm/tool_parsers/tool_strict_level.py diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 9628bc431a..fb70c50248 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -109,18 +109,22 @@ vLLM supports the `tool_choice='none'` option in the chat completion API. When t ## Constrained Decoding Behavior -Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode and the per-tool `strict` field: +Structural-tag parsers resolve call obligation, grammar activation, and argument-schema enforcement separately. With structural-tag enforcement enabled: -| `tool_choice` value | Schema-constrained decoding | Behavior | +| `tool_choice` value | Call obligation | Structural-tag activation | | --- | --- | --- | -| Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. | -| `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. | -| `"auto"` | Only when `strict: true` is set on at least one tool | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`. Without it, the model generates freely and tool calls are extracted from raw text. | -| `"none"` | N/A | No tool calls are produced. | +| Named function | Call the selected function | Always | +| `"required"` | Produce at least one tool call | Always | +| `"auto"` | Tool calls are optional | When at least one tool sets `strict: true`, or `--tool-strict-level` is `function` or `parameter` | +| `"none"` | Tool calling is disabled | Disabled | + +When a structural tag applies, each tool's declared parameter schema is enforced only when that tool sets `strict: true` or the server uses `--tool-strict-level parameter`. Tools with omitted or false `strict` receive broad argument-syntax constraints at levels `auto` and `function`, including for required and named calls. + +For parsers using schema-derived JSON constraints, required and named calls continue to enforce the declared parameter schemas. ### Strict Mode -For `tool_choice="required"` or named function calling, structural-tag constraints are always applied regardless of the `strict` field. For `tool_choice="auto"`, setting `strict: true` on at least one tool opts in to structural-tag constraints; without it, the model generates freely and tool calls are extracted from raw text. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages. +Structural-tag constraints have two layers. The tool-call *envelope* (markup and function name) is constrained whenever a structural tag applies: always for `tool_choice="required"` and named function calling, and for `tool_choice="auto"` once at least one tool sets `strict: true` or the server raises the floor. The *argument schema* of an individual tool is pinned only when that tool sets `strict: true` (or under `--tool-strict-level parameter`); a tool that omits `strict` keeps its arguments unconstrained, even when another tool in the same request is strict. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages. For best compatibility with strict schema enforcement, define tool parameter schemas in the OpenAI strict-schema style: @@ -134,6 +138,22 @@ vLLM also provides a global toggle via the `VLLM_ENFORCE_STRICT_TOOL_CALLING` en VLLM_ENFORCE_STRICT_TOOL_CALLING=false vllm serve ... ``` +### Server-Side Strictness Floor + +Most OpenAI-compatible clients and agent frameworks never set `strict` on their tools, so with `tool_choice="auto"` the model generates tool calls without any grammar and malformed markup can leak into the response. The `--tool-strict-level` option lets the server operator raise the floor for every request that carries tools, independently of what the client declares: + +| Value | Behavior | +| --- | --- | +| `auto` (default) | Follow the request's tool choice and per-tool strictness. Required/named choices activate structural tags; `tool_choice="auto"` activates them when at least one tool sets `strict: true`. | +| `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. | +| `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. | + +```bash +vllm serve ... --tool-strict-level function +``` + +The floor never relaxes a constraint the request would already receive: tools the client marked `strict: true` keep their schemas at every level. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this option. + ## Automatic Function Calling To enable this feature, you should set the following flags: @@ -152,7 +172,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template! !!! note - With `tool_choice="auto"`, schema-level constraint requires both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`. When these conditions are met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. + With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. ### Hermes Models (`hermes`) @@ -162,8 +182,8 @@ All Nous Research Hermes-series models newer than Hermes 2 Pro should be support * `NousResearch/Hermes-2-Theta-*` * `NousResearch/Hermes-3-*` -_Note that the Hermes 2 **Theta** models are known to have degraded tool call quality and capabilities due to the merge -step in their creation_. +*Note that the Hermes 2 **Theta** models are known to have degraded tool call quality and capabilities due to the merge +step in their creation*. Flags: `--tool-call-parser hermes` diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index af9af8b51c..07f0b2dce8 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -134,6 +134,7 @@ impl ChatBackend for HfChatBackend { self.tokenizer.clone(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } @@ -343,6 +344,7 @@ mod tests { let error = match backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { + tool_strict_level: crate::ToolStrictLevel::Auto, tool_call_parser: &ParserSelection::Explicit("json".to_string()), reasoning_parser: &ParserSelection::Auto, }, diff --git a/rust/src/chat/src/backend/mod.rs b/rust/src/chat/src/backend/mod.rs index af4579159e..a0343e53f3 100644 --- a/rust/src/chat/src/backend/mod.rs +++ b/rust/src/chat/src/backend/mod.rs @@ -13,7 +13,7 @@ use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo}; use crate::output::DynChatOutputProcessor; use crate::renderer::DynChatRenderer; use crate::request::ChatRequest; -use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; +use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection, ToolStrictLevel}; pub mod hf; @@ -21,6 +21,7 @@ pub mod hf; pub struct NewChatOutputProcessorOptions<'a> { pub tool_call_parser: &'a ParserSelection, pub reasoning_parser: &'a ParserSelection, + pub tool_strict_level: ToolStrictLevel, } /// Minimal prompt-processing backend needed by `vllm-chat`. diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 69d39ca78e..0273396ebd 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -30,7 +30,7 @@ pub use parser::reasoning::{ ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, }; pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory}; -pub use parser::{ParserSelection, validate_parser_overrides}; +pub use parser::{ParserSelection, ToolStrictLevel, validate_parser_overrides}; pub use reasoning::EffortValue; pub use renderer::hf::ChatTemplateContentFormatOption; pub use renderer::{ @@ -74,6 +74,8 @@ pub struct ChatRequestProcessor { tool_call_parser: ParserSelection, /// Reasoning parser selection used when preparing generation requests. reasoning_parser: ParserSelection, + /// Server-side floor for tool-call structural tags. + tool_strict_level: ToolStrictLevel, } impl ChatRequestProcessor { @@ -85,6 +87,7 @@ impl ChatRequestProcessor { model_dtype: Some(model_dtype), tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Auto, } } @@ -95,6 +98,7 @@ impl ChatRequestProcessor { model_dtype: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Auto, } } @@ -109,6 +113,12 @@ impl ChatRequestProcessor { self } + /// Configure the server-side floor for tool-call structural tags. + pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self { + self.tool_strict_level = tool_strict_level; + self + } + async fn finalize_rendered_prompt( &self, request: &ChatRequest, @@ -207,6 +217,7 @@ impl ChatRequestProcessor { NewChatOutputProcessorOptions { tool_call_parser: &self.tool_call_parser, reasoning_parser: &self.reasoning_parser, + tool_strict_level: self.tool_strict_level, }, )?; let text_request = self.prepare_text_request(request).await?; @@ -255,6 +266,12 @@ impl ChatLlm { self } + /// Set the server-side floor for tool-call structural tags. + pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self { + self.processor.tool_strict_level = tool_strict_level; + self + } + /// Override the effective model dtype used for multimodal tensor encoding. pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self { self.processor.model_dtype = Some(model_dtype); diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index a731153063..9fb3d02b11 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -18,10 +18,10 @@ use self::unified::unified_event_stream; use super::structured::structured_chat_event_stream; use crate::error::Result; use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream}; -use crate::parser::ParserSelection; use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; use crate::parser::unified::UnifiedParserFactory; +use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::{ChatRequest, ChatTool}; use crate::{Error, Result as ChatResult}; @@ -49,6 +49,7 @@ impl DefaultChatOutputProcessor { tokenizer: DynTokenizer, tool_call_parser: &ParserSelection, reasoning_parser: &ParserSelection, + tool_strict_level: ToolStrictLevel, ) -> ChatResult { let parser = if let Some(parser) = Self::resolve_optional_unified_parser( request.tools(), @@ -73,7 +74,11 @@ impl DefaultChatOutputProcessor { Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box }; - apply_structural_tag_constraint(request, parser.structural_tag_builder())?; + apply_structural_tag_constraint( + request, + parser.structural_tag_builder(), + tool_strict_level, + )?; if parser.preserve_special_tokens() { request.decode_options.skip_special_tokens = false; @@ -191,7 +196,7 @@ mod tests { use vllm_tokenizer::test_utils::TestTokenizer; use super::DefaultChatOutputProcessor; - use crate::parser::ParserSelection; + use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::ChatRequest; fn tokenizer() -> Arc { @@ -213,6 +218,7 @@ mod tests { tokenizer(), &selection, &selection, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -227,6 +233,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Auto, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -244,6 +251,7 @@ mod tests { tokenizer(), tool, reasoning, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -258,6 +266,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Explicit("gemma4".to_string()), + ToolStrictLevel::Auto, ) { Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"), Err(error) => error, diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs index 2ef6f08685..116cd53a4a 100644 --- a/rust/src/chat/src/output/default/structural_tag.rs +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -14,19 +14,21 @@ use xgrammar_structural_tag::{ build_structural_tag, }; +use crate::parser::ToolStrictLevel; use crate::request::{ChatRequest, ChatToolChoice}; use crate::{Error, Result as ChatResult}; /// Apply structural tag constraints to the request based on the tool parser's structural tag -/// support and the request's tool choice. +/// support, the request's tool choice, and the server-side strictness floor. pub(super) fn apply_structural_tag_constraint( request: &mut ChatRequest, builder: Option<&dyn StructuralTagBuilder>, + strict_level: ToolStrictLevel, ) -> ChatResult<()> { let Some(builder) = builder else { return Ok(()); }; - let Some(tool_choice) = structural_tag_tool_choice(request) else { + let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else { return Ok(()); }; @@ -34,11 +36,14 @@ pub(super) fn apply_structural_tag_constraint( .tools() .iter() .map(|tool| { + // A tool without an explicit `strict` is non-strict: its call envelope is + // constrained, but its arguments stay free unless the server level is `parameter`. + let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true); ToolParam::Function(FunctionToolParam::new(FunctionDefinition { name: tool.name.clone(), description: tool.description.clone(), parameters: Some(tool.parameters.clone()), - strict: tool.strict, + strict: Some(strict), })) }) .collect::>(); @@ -66,17 +71,21 @@ pub(super) fn apply_structural_tag_constraint( /// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request. /// /// Returns `None` if no structural tag constraints should be applied. -fn structural_tag_tool_choice(request: &ChatRequest) -> Option { +fn structural_tag_tool_choice( + request: &ChatRequest, + strict_level: ToolStrictLevel, +) -> Option { if request.tools().is_empty() { return None; } match request.tool_choice() { - // For `Auto`, only apply the structural tag if there's at least one strict tool. - ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => { - Some(StructuralTagToolChoice::auto()) - } - ChatToolChoice::Auto | ChatToolChoice::None => None, + ChatToolChoice::None => None, + // For `Auto`, apply the structural tag only when the server raises the floor or at + // least one tool opts in with `strict: true`. + ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function + || request.tools().iter().any(|tool| tool.strict == Some(true))) + .then(StructuralTagToolChoice::auto), ChatToolChoice::Required => Some(StructuralTagToolChoice::required()), ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())), @@ -148,8 +157,12 @@ mod tests { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert_eq!(tag["type"], "structural_tag"); @@ -172,8 +185,12 @@ mod tests { }; let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert!(tag.to_string().contains("lookup")); @@ -199,8 +216,12 @@ mod tests { }; let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request).to_string(); assert!(tag.contains("search")); @@ -212,8 +233,12 @@ mod tests { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag decision should succeed"); assert!(request.sampling_params.structured_outputs.is_none()); } @@ -227,8 +252,12 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let params = structured_outputs(&request); assert!(params.constraint.is_structural_tag()); @@ -242,8 +271,12 @@ mod tests { let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert_eq!(tag["type"], "structural_tag"); @@ -259,8 +292,12 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let params = structured_outputs(&request); assert!(params.constraint.is_structural_tag()); @@ -279,8 +316,12 @@ mod tests { ); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request).to_string(); assert!(tag.contains("lookup")); @@ -292,8 +333,12 @@ mod tests { let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag decision should succeed"); assert!(request.sampling_params.structured_outputs.is_none()); } @@ -307,10 +352,127 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag decision should succeed"); let params = structured_outputs(&request); assert!(params.constraint.is_json_object()); } + + fn build(tool_choice: ChatToolChoice, tools: Vec, level: ToolStrictLevel) -> ChatRequest { + let mut request = request(tool_choice, tools); + let parser = qwen3_coder_parser(request.tools()); + + apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level) + .expect("structural tag decision should succeed"); + request + } + + fn tag_string(request: &ChatRequest) -> String { + structural_tag_value(request).to_string() + } + + #[test] + fn function_level_constrains_auto_without_strict_tools() { + let request = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Function, + ); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn absent_strict_is_non_strict() { + let unset = build( + ChatToolChoice::Required, + vec![chat_tool("search", None)], + ToolStrictLevel::Auto, + ); + let non_strict = build( + ChatToolChoice::Required, + vec![chat_tool("search", Some(false))], + ToolStrictLevel::Auto, + ); + let strict = build( + ChatToolChoice::Required, + vec![chat_tool("search", Some(true))], + ToolStrictLevel::Auto, + ); + + assert_eq!(tag_string(&unset), tag_string(&non_strict)); + assert_ne!(tag_string(&unset), tag_string(&strict)); + } + + #[test] + fn strict_tool_does_not_pin_its_neighbour() { + let mixed = build( + ChatToolChoice::Auto, + vec![chat_tool("weather", Some(true)), chat_tool("search", None)], + ToolStrictLevel::Auto, + ); + let explicit = build( + ChatToolChoice::Auto, + vec![ + chat_tool("weather", Some(true)), + chat_tool("search", Some(false)), + ], + ToolStrictLevel::Auto, + ); + + assert_eq!(tag_string(&mixed), tag_string(&explicit)); + } + + #[test] + fn parameter_level_pins_argument_schemas() { + let pinned = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Parameter, + ); + let strict = build( + ChatToolChoice::Auto, + vec![chat_tool("search", Some(true))], + ToolStrictLevel::Auto, + ); + let envelope_only = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Function, + ); + + assert_eq!(tag_string(&pinned), tag_string(&strict)); + assert_ne!(tag_string(&pinned), tag_string(&envelope_only)); + } + + #[test] + fn tool_strict_level_parses_case_insensitively() { + assert_eq!("AUTO".parse::(), Ok(ToolStrictLevel::Auto)); + assert_eq!( + "PARAMETER".parse::(), + Ok(ToolStrictLevel::Parameter) + ); + assert_eq!( + "function".parse::(), + Ok(ToolStrictLevel::Function) + ); + assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto); + assert_eq!( + serde_json::to_value(ToolStrictLevel::Auto).unwrap(), + json!("auto") + ); + assert_eq!( + serde_json::from_value::(json!("auto")).unwrap(), + ToolStrictLevel::Auto + ); + assert!("off".parse::().is_err()); + assert!("strict".parse::().is_err()); + } } diff --git a/rust/src/chat/src/parser/mod.rs b/rust/src/chat/src/parser/mod.rs index c10a4f11bf..5a86b24934 100644 --- a/rust/src/chat/src/parser/mod.rs +++ b/rust/src/chat/src/parser/mod.rs @@ -57,6 +57,66 @@ impl ParserSelection { } } +/// Server-side floor for tool-call structural tags (`--tool-strict-level`). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + DeserializeFromStr, + SerializeDisplay, +)] +pub enum ToolStrictLevel { + /// Derive tool constraints from the request's tool choice and per-tool strictness. + #[default] + Auto, + /// Constrain the tool-call envelope for every request with tools. + Function, + /// Additionally pin argument schemas, as if every tool were `strict: true`. + Parameter, +} + +impl ToolStrictLevel { + pub const AUTO_LITERAL: &str = "auto"; + pub const FUNCTION_LITERAL: &str = "function"; + pub const PARAMETER_LITERAL: &str = "parameter"; +} + +impl FromStr for ToolStrictLevel { + type Err = String; + + fn from_str(value: &str) -> Result { + if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) { + Ok(Self::Auto) + } else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) { + Ok(Self::Function) + } else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) { + Ok(Self::Parameter) + } else { + Err(format!( + "unknown tool strict level {value:?}; expected one of {}, {}, {}", + Self::AUTO_LITERAL, + Self::FUNCTION_LITERAL, + Self::PARAMETER_LITERAL + )) + } + } +} + +impl fmt::Display for ToolStrictLevel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Auto => Self::AUTO_LITERAL, + Self::Function => Self::FUNCTION_LITERAL, + Self::Parameter => Self::PARAMETER_LITERAL, + }) + } +} + /// Validate explicit parser override names without starting request processing. pub fn validate_parser_overrides( tool_call_parser: &ParserSelection, diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index ed96134cdc..ab366b8827 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -227,6 +227,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 5669e255e0..db3559c44f 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -717,6 +717,7 @@ async fn parse_completion( let processor = backends.chat_backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { + tool_strict_level: vllm_chat::ToolStrictLevel::Auto, tool_call_parser: &case.tool_call_parser, reasoning_parser: &case.reasoning_parser, }, diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index cda2e6c1a8..5bdf66da57 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -24,6 +24,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as}; use thiserror_ext::AsReport as _; use uuid::Uuid; use vllm_chat::GenerationConfigMode; +use vllm_chat::ToolStrictLevel; use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_engine_core_client::TransportMode; use vllm_managed_engine::ManagedEngineConfig; @@ -134,6 +135,10 @@ pub struct RenderArgs { /// `none` to disable parsing. #[arg(long, default_value_t)] reasoning_parser: ParserSelection, + /// Server-side floor for structural-tag based tool calling: `auto`, + /// `function`, or `parameter`. + #[arg(long, default_value_t)] + tool_strict_level: ToolStrictLevel, /// Select the native chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] renderer: RendererSelection, @@ -171,6 +176,7 @@ impl RenderArgs { port: self.port, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(), @@ -241,6 +247,14 @@ pub struct SharedRuntimeArgs { #[arg(long, default_value_t)] #[serde(default = "default_py_bootstrap_parser_selection")] pub reasoning_parser: ParserSelection, + /// Server-side floor for structural-tag based tool calling, applied on + /// top of the per-tool `strict` field: `auto` follows the request's + /// tool choice and per-tool strictness, + /// `function` constrains the tool-call envelope for every request with + /// tools, `parameter` additionally pins argument schemas. + #[arg(long, default_value_t)] + #[serde(default)] + pub tool_strict_level: ToolStrictLevel, /// Select the chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] #[serde(default, rename = "tokenizer_mode")] @@ -517,6 +531,7 @@ impl SharedRuntimeArgs { listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd }, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, language_model_only: self.language_model_only, chat_template: self.chat_template, @@ -574,6 +589,7 @@ impl SharedRuntimeArgs { listener_mode, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, language_model_only: self.language_model_only, chat_template: self.chat_template, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 5cc6b19214..f7f6d0a658 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -184,6 +184,7 @@ fn serve_args_forward_python_flags_with_separator() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1023,6 +1024,7 @@ fn frontend_args_accept_json() { engine_ready_timeout_secs: 600, tool_call_parser: None, reasoning_parser: None, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1693,6 +1695,7 @@ fn serve_args_accept_handshake_aliases() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1850,6 +1853,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { }, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, @@ -1942,6 +1946,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { }, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, @@ -2055,6 +2060,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present }, tool_call_parser: None, reasoning_parser: None, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index ad3872843a..3de227babd 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -16,6 +16,7 @@ use clap::Parser; use futures::StreamExt as _; use tokio_util::sync::CancellationToken; use tracing_subscriber::EnvFilter; +use vllm_chat::ToolStrictLevel; use vllm_engine_core_client::TransportMode; use vllm_server::{ ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig, @@ -70,6 +71,7 @@ async fn main() -> Result<()> { }, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Auto, renderer: RendererSelection::Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 4c337adbef..afcfaeccfb 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -14,6 +14,7 @@ use serde_json::Value; use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_chat::{ ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection, + ToolStrictLevel, }; use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode}; use vllm_text::backend::hf::HfOverrides; @@ -229,6 +230,8 @@ pub struct Config { pub tool_call_parser: ParserSelection, /// Reasoning parser selection. pub reasoning_parser: ParserSelection, + /// Server-side floor for tool-call structural tags. + pub tool_strict_level: ToolStrictLevel, /// Chat renderer selection. pub renderer: RendererSelection, /// Disable frontend-side multimodal preprocessing and render the model as diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index f4b35c9df4..18b6e6eb29 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -244,6 +244,7 @@ impl ChatBackend for FakeTextBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } @@ -292,6 +293,7 @@ impl ChatBackend for FakeMultimodalBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 49d3d57f32..91564db73c 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -139,7 +139,8 @@ async fn build_state(config: &Config) -> Result> { let chat = ChatLlm::new(text, chat_backend) .with_tool_call_parser(config.tool_call_parser.clone()) - .with_reasoning_parser(config.reasoning_parser.clone()); + .with_reasoning_parser(config.reasoning_parser.clone()) + .with_tool_strict_level(config.tool_strict_level); let state = Arc::new( AppState::new(served_model_names, chat) diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs index 4fc55fe60b..50485e26a4 100644 --- a/rust/src/server/src/render.rs +++ b/rust/src/server/src/render.rs @@ -10,7 +10,7 @@ use tokio_util::sync::CancellationToken; use tracing::info; use vllm_chat::{ ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions, - ParserSelection, RendererSelection, load_model_backends, + ParserSelection, RendererSelection, ToolStrictLevel, load_model_backends, }; use vllm_text::TextRequestProcessor; use vllm_text::backend::hf::HfOverrides; @@ -32,6 +32,7 @@ pub struct RenderConfig { pub port: u16, pub tool_call_parser: ParserSelection, pub reasoning_parser: ParserSelection, + pub tool_strict_level: ToolStrictLevel, pub renderer: RendererSelection, pub chat_template: Option, pub default_chat_template_kwargs: HashMap, @@ -88,10 +89,12 @@ async fn build_state(config: &RenderConfig) -> Result> { let max_model_len = config.max_model_len.unwrap_or(u32::MAX); let text = TextRequestProcessor::new(loaded.text_backend, max_model_len) .with_max_logprobs(config.max_logprobs); - let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections( - config.tool_call_parser.clone(), - config.reasoning_parser.clone(), - ); + let chat = ChatRequestProcessor::render_only(loaded.chat_backend) + .with_parser_selections( + config.tool_call_parser.clone(), + config.reasoning_parser.clone(), + ) + .with_tool_strict_level(config.tool_strict_level); Ok(Arc::new(RenderState { model: config.model.clone(), served_model_names, @@ -154,6 +157,7 @@ mod tests { shutdown.cancel(); let error = serve_render( RenderConfig { + tool_strict_level: ToolStrictLevel::Auto, model: "test-model".to_string(), revision: None, hf_overrides: Default::default(), diff --git a/rust/src/server/src/routes/http_client_tests.rs b/rust/src/server/src/routes/http_client_tests.rs index a0e0872e64..762d974b9b 100644 --- a/rust/src/server/src/routes/http_client_tests.rs +++ b/rust/src/server/src/routes/http_client_tests.rs @@ -168,6 +168,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index c27ac20fad..76d9ab8627 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -517,6 +517,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 06c8fb6129..c216b398cc 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json from types import SimpleNamespace from unittest.mock import MagicMock @@ -33,9 +34,11 @@ from vllm.tool_parsers.structural_tag_registry import ( SUPPORTED_STRUCTURAL_TAG_MODELS, VLLM_BUILTIN_STRUCTURAL_TAG_MODELS, XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS, + ToolChoice, get_function_parameters, get_model_structural_tag, ) +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel @pytest.fixture @@ -231,11 +234,11 @@ def test_get_model_structural_tag_supports_all_xgrammar_builtins( def test_get_model_structural_tag_supports_vllm_hermes( - sample_tools: list[ChatCompletionToolsParam], + sample_tools_strict: list[ChatCompletionToolsParam], ): tag = get_model_structural_tag( model="hermes", - tools=sample_tools, + tools=sample_tools_strict, tool_choice="required", reasoning=False, ) @@ -273,6 +276,22 @@ def test_get_model_structural_tag_supports_vllm_hermes( assert content["json_schema"] == expected_schema +def test_hermes_required_without_strict_leaves_arguments_free( + sample_tools: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model="hermes", + tools=sample_tools, + tool_choice="required", + reasoning=False, + ) + + assert isinstance(tag, StructuralTag) + tags = tag.model_dump()["format"]["tags"] + assert tags + assert all(tag_dump["content"]["json_schema"] is True for tag_dump in tags) + + def test_hermes_required_tool_calls_use_empty_separator(): tools = [ ChatCompletionToolsParam( @@ -361,10 +380,19 @@ def _k3_tools(*calls: str) -> str: return f"{_K3_TOOLS_OPEN}{''.join(calls)}{_K3_TOOLS_CLOSE}" -def _k3_grammar(tool_choice, tools=None): +def _as_strict(tools: list[ChatCompletionToolsParam]) -> list[ChatCompletionToolsParam]: + for tool in tools: + tool.function.strict = True + return tools + + +def _k3_grammar(tool_choice, tools=None, *, strict: bool = True): + """Build the K3 grammar; tools are strict by default so the typed + argument channel is exercised (an unset ``strict`` leaves arguments free).""" + tools = tools if tools is not None else _k3_tools_by_name() tag = get_model_structural_tag( model="kimi_k3", - tools=tools if tools is not None else _k3_tools_by_name(), + tools=_as_strict(tools) if strict else tools, tool_choice=tool_choice, reasoning=False, ) @@ -467,6 +495,18 @@ def test_kimi_k3_required_rejects_invalid(body: str): assert not _is_grammar_accept_string(_k3_grammar("required"), body) +def test_kimi_k3_required_without_strict_leaves_arguments_free(): + # An unset ``strict`` pins the call envelope only: a value the schema + # would reject is accepted, and the declared-required argument may be + # omitted. + grammar = _k3_grammar("required", strict=False) + body = _k3_response() + _k3_tools( + _k3_call("get_weather", _k3_arg("days", "number", "abc")) + ) + assert _is_grammar_accept_string(grammar, body) + assert not _is_grammar_accept_string(_k3_grammar("required"), body) + + def test_kimi_k3_schema_without_required_accepts_empty_call(): tools = [ ChatCompletionToolsParam( @@ -829,3 +869,209 @@ def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call(): response_only = _k3_response("no call here") assert _is_grammar_accept_string(grammar, ok) assert not _is_grammar_accept_string(grammar, response_only) + + +def _pins_argument_schema(tag: StructuralTag) -> bool: + return '"json_schema": {' in json.dumps(tag.model_dump(), ensure_ascii=False) + + +def _dumped(tag: StructuralTag) -> str: + return json.dumps(tag.model_dump(), ensure_ascii=False) + + +def test_tool_strict_level_auto_is_the_default( + sample_tools: list[ChatCompletionToolsParam], +): + """Auto + no strict tool gets no tag unless the operator raises the floor.""" + assert ( + get_model_structural_tag( + model="deepseek_v4", + tools=sample_tools, + tool_choice="auto", + reasoning=False, + ) + is None + ) + + +@pytest.mark.parametrize("model", sorted(SUPPORTED_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_function_lifts_auto_gate( + model: str, + sample_tools: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + + assert tag is not None + + +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_function_pins_envelope_only( + model: str, + sample_tools: list[ChatCompletionToolsParam], +): + """The request's own tools stay untouched; unset ``strict`` stays free.""" + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + + assert tag is not None + assert not _pins_argument_schema(tag) + assert all(tool.function.strict is None for tool in sample_tools) + + +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_parameter_pins_argument_schemas( + model: str, + sample_tools: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +def test_tool_strict_level_function_keeps_client_strict_tools_strict( + sample_tools_strict: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model="deepseek_v4", + tools=sample_tools_strict, + tool_choice="auto", + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_absent_strict_is_not_pinned_next_to_a_strict_tool(model: str): + """A strict tool must not drag a neighbour with unset ``strict`` into + schema enforcement.""" + weather = ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + ) + search = ChatCompletionToolsParam( + type="function", + function={ + "name": "search", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + ) + tag = get_model_structural_tag( + model=model, + tools=[weather, search], + tool_choice="auto", + reasoning=False, + ) + + assert tag is not None + dumped = _dumped(tag) + assert '"city"' in dumped + assert '"query"' not in dumped + + +@pytest.mark.parametrize( + "tool_choice", + [ + "required", + ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="get_weather") + ), + ], +) +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_forced_tool_choice_pins_schema_only_for_strict_tools( + model: str, + tool_choice: ToolChoice, + sample_tools: list[ChatCompletionToolsParam], + sample_tools_strict: list[ChatCompletionToolsParam], +): + """Required / named always constrain the call; the argument schema + follows the per-tool ``strict`` (or the parameter level).""" + free = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice=tool_choice, + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + pinned = get_model_structural_tag( + model=model, + tools=sample_tools_strict, + tool_choice=tool_choice, + reasoning=False, + ) + floored = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice=tool_choice, + reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, + ) + + assert free is not None and not _pins_argument_schema(free) + assert pinned is not None and _pins_argument_schema(pinned) + assert floored is not None and _pins_argument_schema(floored) + + +def test_tool_strict_level_parameter_overrides_client_strict_false(): + tools = [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + "strict": False, + }, + ) + ] + tag = get_model_structural_tag( + model="deepseek_v4", + tools=tools, + tool_choice="required", + reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +def test_tool_strict_level_from_name(): + assert ToolStrictLevel.from_name("AUTO") is ToolStrictLevel.AUTO + assert ToolStrictLevel.from_name("Parameter") is ToolStrictLevel.PARAMETER + with pytest.raises(ValueError, match="expected one of auto, function, parameter"): + ToolStrictLevel.from_name("strict") + with pytest.raises(ValueError, match="expected one of auto, function, parameter"): + ToolStrictLevel.from_name("off") diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index e90d70e289..6cc1271add 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -112,6 +112,7 @@ class AnthropicServingMessages(OpenAIServingChat): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, @@ -127,6 +128,7 @@ class AnthropicServingMessages(OpenAIServingChat): return_tokens_as_token_ids=return_tokens_as_token_ids, reasoning_parser=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, tool_parser=tool_parser, enable_prompt_tokens_details=enable_prompt_tokens_details, enable_force_include_usage=enable_force_include_usage, diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index 7885586452..63d00f0c57 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -249,6 +249,7 @@ class CohereServingChatV2(OpenAIServingChat): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, @@ -265,6 +266,7 @@ class CohereServingChatV2(OpenAIServingChat): return_tokens_as_token_ids=return_tokens_as_token_ids, reasoning_parser=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, tool_parser=tool_parser, enable_prompt_tokens_details=enable_prompt_tokens_details, enable_force_include_usage=enable_force_include_usage, diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index ac995a4f03..189c4e6779 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -112,6 +112,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, tool_server=state.tool_server, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, @@ -137,6 +138,7 @@ async def init_generate_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, @@ -178,6 +180,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, @@ -198,6 +201,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, diff --git a/vllm/entrypoints/launchers/api_server/app_state.py b/vllm/entrypoints/launchers/api_server/app_state.py index 09bb0ec630..d33be65949 100644 --- a/vllm/entrypoints/launchers/api_server/app_state.py +++ b/vllm/entrypoints/launchers/api_server/app_state.py @@ -95,6 +95,7 @@ async def init_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, @@ -111,6 +112,7 @@ async def init_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 71ae94c106..488a7f2c43 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -20,6 +20,7 @@ from vllm.entrypoints.chat_utils import ( ) from vllm.entrypoints.openai.models.protocol import LoRAModulePath from vllm.tool_parsers import ToolParserManager +from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName from vllm.utils.argparse_utils import FlexibleArgumentParser from .utils.constants import ( @@ -116,6 +117,13 @@ class BaseFrontendArgs: """Special the tool parser plugin write to parse the model-generated tool into OpenAI API format, the name register in this plugin can be used in `--tool-call-parser`.""" + tool_strict_level: ToolStrictLevelName = "auto" + """Server-side floor for structural-tag based tool calling, applied on top + of the per-tool `strict` field. `auto` follows the request's tool choice + and per-tool strictness; `function` constrains the + tool-call envelope (markup and function name) for every request with + tools; `parameter` additionally pins argument schemas, as if every tool + were `strict: true`.""" tool_server: str | None = None """Comma-separated list of host:port pairs (IPv4, IPv6, or hostname). Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for diff --git a/vllm/entrypoints/launchers/render/app_state.py b/vllm/entrypoints/launchers/render/app_state.py index 33988443d1..1d242977be 100644 --- a/vllm/entrypoints/launchers/render/app_state.py +++ b/vllm/entrypoints/launchers/render/app_state.py @@ -60,6 +60,7 @@ async def init_render_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, @@ -76,6 +77,7 @@ async def init_render_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 51c0af8e8f..3e2d8f7369 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -132,6 +132,7 @@ class OpenAIServingChat(GenerateBaseServing): enable_auto_tools: bool = False, exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, @@ -161,6 +162,7 @@ class OpenAIServingChat(GenerateBaseServing): tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index ee4f7d55ca..8108c5f775 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -109,6 +109,7 @@ class OpenAIServingResponses(GenerateBaseServing): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "auto", tool_server: ToolServer | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, @@ -135,6 +136,7 @@ class OpenAIServingResponses(GenerateBaseServing): tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index af448e0951..0e84e710ed 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -36,6 +36,7 @@ from vllm.tool_parsers.streaming import ( extract_named_tool_call_streaming, extract_required_tool_call_streaming, ) +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel logger = init_logger(__name__) @@ -95,6 +96,8 @@ class Parser: # Subclasses should override these if they use specific parser classes reasoning_parser_cls: type[ReasoningParser] | None = None tool_parser_cls: type[ToolParser] | None = None + # Server-side floor for tool-call structural tags (--tool-strict-level). + tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO def __init__( self, @@ -415,6 +418,7 @@ class DelegatingParser(Parser): structure_tag = self._tool_parser.get_structural_tag( request, reasoning=False, + strict_level=self.tool_strict_level, ) if structure_tag is None: return request diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index a8efd3f0ea..41b27967a6 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -6,6 +6,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from vllm.logger import init_logger +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel if TYPE_CHECKING: from vllm.parser.abstract_parser import Parser @@ -77,6 +78,7 @@ class ParserManager: enable_auto_tools: bool = False, model_name: str | None = None, is_harmony: bool = False, + tool_strict_level: str = "auto", ) -> type[Parser] | None: """Get a Parser that handles both reasoning and tool parsing. @@ -89,6 +91,8 @@ class ParserManager: model_name: The model name for parser-specific warnings. is_harmony: Whether the selected model uses the Harmony format. If True, HarmonyParser is always returned. + tool_strict_level: Server-side floor for tool-call structural + tags (``--tool-strict-level``). Returns: A Parser class, or None if neither parser is specified. @@ -105,11 +109,14 @@ class ParserManager: if reasoning_parser_cls is None and tool_parser_cls is None: return None + strict_level = ToolStrictLevel.from_name(tool_strict_level) + if is_harmony: from vllm.parser.harmony import HarmonyParser HarmonyParser.reasoning_parser_cls = reasoning_parser_cls HarmonyParser.tool_parser_cls = tool_parser_cls + HarmonyParser.tool_strict_level = strict_level return HarmonyParser if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3": @@ -121,6 +128,7 @@ class ParserManager: class _KimiK3Parser(KimiK3Parser): reasoning_parser_cls = r_cls tool_parser_cls = t_cls + tool_strict_level = strict_level return _KimiK3Parser @@ -136,6 +144,7 @@ class ParserManager: class _CohereCommandParser(CohereCommandParser): reasoning_parser_cls = r_cls tool_parser_cls = t_cls + tool_strict_level = strict_level return _CohereCommandParser @@ -147,5 +156,6 @@ class ParserManager: class _Parser(DelegatingParser): reasoning_parser_cls = r_cls tool_parser_cls = t_cls + tool_strict_level = strict_level return _Parser diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 5f30b552ba..05f2710afb 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -54,6 +54,7 @@ class OnlineDerenderer: exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, reasoning_parser: str | None = None, + tool_strict_level: str = "auto", default_chat_template_kwargs: dict[str, Any] | None = None, log_error_stack: bool = False, ) -> None: @@ -68,6 +69,7 @@ class OnlineDerenderer: tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=model_config.model, is_harmony=self.use_harmony, ) diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 2ab5725b73..f4d21dedad 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -127,6 +127,7 @@ class OnlineRenderer: exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, reasoning_parser: str | None = None, + tool_strict_level: str = "auto", default_chat_template_kwargs: dict[str, Any] | None = None, log_error_stack: bool = False, ) -> None: @@ -141,6 +142,7 @@ class OnlineRenderer: tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=model_config.model, is_harmony=self.use_harmony, ) diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index 3847afa9b2..953782daa9 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -30,6 +30,7 @@ from vllm.sampling_params import ( StructuredOutputsParams, ) from vllm.tokenizers import TokenizerLike +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel from vllm.tool_parsers.utils import Tool, get_json_schema_from_tools from vllm.utils.collection_utils import is_list_of from vllm.utils.import_utils import import_plugin @@ -168,6 +169,7 @@ class ToolParser: request: ChatCompletionRequest | ResponsesRequest, *, reasoning: bool = False, + strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, ): if self.structural_tag_model is None: return None @@ -180,6 +182,7 @@ class ToolParser: tools=request.tools, tool_choice=request.tool_choice, reasoning=reasoning, + strict_level=strict_level, ) def extract_tool_calls( diff --git a/vllm/tool_parsers/hy_v4_tool_parser.py b/vllm/tool_parsers/hy_v4_tool_parser.py index c2f82fc4a7..b9c857dbd9 100644 --- a/vllm/tool_parsers/hy_v4_tool_parser.py +++ b/vllm/tool_parsers/hy_v4_tool_parser.py @@ -25,6 +25,7 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import ToolParser +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel from vllm.tool_parsers.utils import partial_tag_overlap if TYPE_CHECKING: @@ -994,6 +995,7 @@ class HYV4ToolParser(ToolParser): request: ChatCompletionRequest | ResponsesRequest, *, reasoning: bool = False, + strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, ) -> StructuralTag | None: """Build a structural tag matching HYV4's tool tokens. @@ -1009,6 +1011,7 @@ class HYV4ToolParser(ToolParser): Args: request: The request being adjusted. reasoning: Whether the grammar also covers the reasoning phase. + strict_level: Server-side floor from ``--tool-strict-level``. Returns: The structural tag, or None when structural tagging does not apply. @@ -1041,6 +1044,7 @@ class HYV4ToolParser(ToolParser): tool_choice=request.tool_choice, reasoning=reasoning, token_suffix=self._extractor.token_suffix, + strict_level=strict_level, ) except Exception: logger.warning( diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 295d5844b4..5d65b186fe 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -35,6 +35,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, ChatCompletionToolsParam, ) +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel ToolChoice: TypeAlias = ( Literal["none", "auto", "required"] @@ -94,15 +95,57 @@ def register_vllm_structural_tag( return decorator +def _tool_is_strict(tool: ChatCompletionToolsParam | ResponsesTool) -> bool: + if isinstance(tool, FunctionTool): + return tool.strict is True + if isinstance(tool, ChatCompletionToolsParam): + return tool.function.strict is True + return False + + def _any_tool_strict( tools: Sequence[ChatCompletionToolsParam | ResponsesTool], ) -> bool: - for tool in tools: - if isinstance(tool, FunctionTool) and tool.strict is True: - return True - if isinstance(tool, ChatCompletionToolsParam) and tool.function.strict is True: - return True - return False + return any(_tool_is_strict(tool) for tool in tools) + + +def _with_tool_strict( + tool: ChatCompletionToolsParam | ResponsesTool, + strict: bool, +) -> ChatCompletionToolsParam | ResponsesTool: + """Return a copy of ``tool`` with ``strict`` set, leaving the request's alone.""" + if isinstance(tool, FunctionTool): + return tool.model_copy(update={"strict": strict}) + if isinstance(tool, ChatCompletionToolsParam): + return tool.model_copy( + update={"function": tool.function.model_copy(update={"strict": strict})} + ) + return tool + + +def _resolve_tool_strictness( + tools: Sequence[ChatCompletionToolsParam | ResponsesTool], + tool_choice: ToolChoice, + strict_level: ToolStrictLevel, +) -> Sequence[ChatCompletionToolsParam | ResponsesTool] | None: + """Decide whether a structural tag applies and pin each tool's ``strict``. + + A tool without an explicit ``strict`` is treated as non-strict: its call + envelope is still constrained, but its arguments stay free unless the + server level is PARAMETER. ``None`` means no structural tag. + """ + if ( + tool_choice == "auto" + and strict_level == ToolStrictLevel.AUTO + and not _any_tool_strict(tools) + ): + return None + return [ + _with_tool_strict( + tool, strict_level >= ToolStrictLevel.PARAMETER or _tool_is_strict(tool) + ) + for tool in tools + ] def get_model_structural_tag( @@ -111,12 +154,14 @@ def get_model_structural_tag( tool_choice: ToolChoice, reasoning: bool, token_suffix: str = "", + strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, ) -> StructuralTag | None: """Build a structural tag with xgrammar's builtin model templates.""" if not tools or tool_choice == "none": return None - if tool_choice == "auto" and not _any_tool_strict(tools): + tools = _resolve_tool_strictness(tools, tool_choice, strict_level) + if tools is None: return None dumped_tools = [_dump_tool_for_xgrammar(tool) for tool in tools] diff --git a/vllm/tool_parsers/tool_strict_level.py b/vllm/tool_parsers/tool_strict_level.py new file mode 100644 index 0000000000..03a7d8b170 --- /dev/null +++ b/vllm/tool_parsers/tool_strict_level.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import enum +from typing import Literal + +ToolStrictLevelName = Literal["auto", "function", "parameter"] + + +class ToolStrictLevel(enum.IntEnum): + """Server-side floor for tool-call structural tags (``--tool-strict-level``). + + AUTO: derive call constraints from ``tool_choice`` and argument-schema + enforcement from each tool's explicit ``strict`` field. + FUNCTION: constrain the tool-call envelope for every request with tools. + PARAMETER: additionally pin argument schemas, as if every tool were + ``strict: true``. + """ + + AUTO = 0 + FUNCTION = 1 + PARAMETER = 2 + + @classmethod + def from_name(cls, name: str) -> "ToolStrictLevel": + try: + return cls[name.strip().upper()] + except KeyError: + expected = ", ".join(level.name.lower() for level in cls) + raise ValueError( + f"Unknown tool strict level {name!r}; expected one of {expected}." + ) from None From 39e33db7f3b10ebd3aca6a0008f2990978f118e7 Mon Sep 17 00:00:00 2001 From: lazymio Date: Fri, 18 Sep 2026 13:11:24 +0800 Subject: [PATCH 079/143] [Frontend] Fix the parsing of missing `string=` in DeepSeek V4 (#56271) Co-authored-by: Claude Opus 5 Co-authored-by: Bugen Zhao Signed-off-by: lazymio --- .../src/tool/deepseek_dsml/deepseek_v32.rs | 28 ++++++++ .../src/tool/deepseek_dsml/deepseek_v4.rs | 23 ++++++ .../tool/deepseek_dsml/deepseek_v41/tests.rs | 22 ++++++ rust/src/parser/src/tool/deepseek_dsml/mod.rs | 11 +-- tests/parser/engine/test_deepseek_v4.py | 70 +++++++++++++++++++ tests/parser/engine/test_deepseek_v41.py | 2 + vllm/parser/deepseek_v4.py | 32 +++++---- vllm/parser/deepseek_v41.py | 15 +--- 8 files changed, 174 insertions(+), 29 deletions(-) diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs index 632eda600f..9bbe39367b 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs @@ -193,6 +193,34 @@ mod tests { ); } + #[test] + fn deepseek_v32_parse_complete_keeps_parameter_without_string_attr() { + // The model sometimes omits `string="..."`; the parameter must still + // be kept and converted through the schema, like `string="false"`. + let mut parser = DeepSeekV32ToolParser::new(&test_tools()); + let output = parser + .parse_complete( + "<|DSML|function_calls>\n\ + <|DSML|invoke name=\"convert\">\n\ + <|DSML|parameter name=\"whole\">5.0\n\ + <|DSML|parameter name=\"flag\" >true\n\ + <|DSML|parameter name=\"payload\" string=\"true\">{\"nested\":true}\n\ + \n\ + ", + ) + .unwrap(); + + assert_eq!(output.calls().len(), 1); + assert_eq!( + serde_json::from_str::(&output.calls()[0].arguments).unwrap(), + json!({ + "whole": 5.0, + "flag": true, + "payload": "{\"nested\":true}", + }) + ); + } + #[test] fn deepseek_v32_parse_complete_preserves_raw_closing_tag_text_in_parameter_value() { let mut parser = DeepSeekV32ToolParser::new(&test_tools()); diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs index b65df235ca..032cfbe5e2 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs @@ -147,6 +147,29 @@ mod tests { ); } + #[test] + fn deepseek_v4_streaming_keeps_parameter_without_string_attr() { + let mut parser = DeepSeekV4ToolParser::new(&test_tools()); + let output = collect_stream( + &mut parser, + &[ + "<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n", + "<|DSML|parameter name=\"loc", + "ation\">Par", + "is\n", + "<|DSML|parameter name=\"date\" string=\"true\">tomorrow\n", + "\n", + ], + ); + + assert!(output.normal_text().is_empty()); + assert_eq!(output.calls().len(), 1); + assert_eq!( + serde_json::from_str::(&output.calls()[0].arguments).unwrap(), + json!({ "location": "Paris", "date": "tomorrow" }) + ); + } + #[test] fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { assert_tool_framing_preserves_body_whitespace::( diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs index d40f82d533..5a6758f5c9 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs @@ -71,6 +71,28 @@ fn spaced_dsml_streaming_preserves_shared_schema_coercion() { } } +#[test] +fn spaced_dsml_keeps_parameter_without_string_attr() { + let wire = concat!( + "<|DSML| calls>\n", + "<|DSML| invoke name=\"lookup\">\n", + "<|DSML| parameter name=\"query\">value\n", + "<|DSML| parameter name=\"limit\">2\n", + "\n", + "", + ); + let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); + + let output = collect_stream(parser.as_mut(), &[wire]); + + assert!(output.normal_text().is_empty()); + assert_eq!(output.calls().len(), 1); + assert_eq!( + serde_json::from_str::(&output.calls()[0].arguments).unwrap(), + json!({ "query": "value", "limit": 2 }) + ); +} + #[test] fn v4_tags_remain_text_in_v41_dialect() { let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); diff --git a/rust/src/parser/src/tool/deepseek_dsml/mod.rs b/rust/src/parser/src/tool/deepseek_dsml/mod.rs index 01a3fbe282..9c7dd20331 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/mod.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use winnow::ascii::{multispace0 as ws0, multispace1 as ws1}; -use winnow::combinator::{alt, delimited, eof, repeat, seq, terminated}; +use winnow::combinator::{alt, delimited, eof, opt, preceded, repeat, seq, terminated}; use winnow::prelude::*; use winnow::stream::Partial; use winnow::token::{literal, rest, take_until}; @@ -293,13 +293,16 @@ fn parse_invoke_params(invoke_body: &str, tokens: DsmlTokens) -> ModalResult ModalResult { seq! {DsmlParameter { _: literal(tokens.parameter_start), _: ws1, name: name_attr.map(|name: &str| name.to_string()), - _: ws1, - is_string: string_attr.map(|value| value == "true"), + is_string: opt(preceded(ws1, string_attr)).map(|value| value == Some("true")), _: ws0, _: ">", value: take_until(0.., tokens.parameter_end).map(str::to_string), @@ -315,7 +318,7 @@ fn name_attr<'i>(input: &mut &'i str) -> ModalResult<&'i str> { /// Parse a string attribute. fn string_attr<'i>(input: &mut &'i str) -> ModalResult<&'i str> { - delimited("string=\"", alt(("true", "false")), "\"").parse_next(input) + delimited("string=\"", take_until(0.., "\""), "\"").parse_next(input) } /// Parse a DSML name attribute. diff --git a/tests/parser/engine/test_deepseek_v4.py b/tests/parser/engine/test_deepseek_v4.py index 7cbf9791a7..2cf3a8bb85 100644 --- a/tests/parser/engine/test_deepseek_v4.py +++ b/tests/parser/engine/test_deepseek_v4.py @@ -119,6 +119,35 @@ class TestArgConverter: result = json.loads(_dsml_arg_converter("", partial=False)) assert result == {} + def _bare(self, name: str, value: str) -> str: + return f'<|DSML|parameter name="{name}">{value}{_PARAM_CLOSE}' + + def test_missing_string_attribute_keeps_literal(self): + """The model sometimes omits ``string``; the parameter must survive.""" + raw = self._bare("name", "alpha") + assert json.loads(_dsml_arg_converter(raw, partial=False)) == {"name": "alpha"} + + def test_missing_string_attribute_parses_json(self): + raw = self._bare("count", "7") + self._bare("opts", '{"k": [1]}') + assert json.loads(_dsml_arg_converter(raw, partial=False)) == { + "count": 7, + "opts": {"k": [1]}, + } + + def test_missing_string_attribute_alongside_annotated(self): + raw = self._bare("a", "x") + _param("b", "true", "42") + assert json.loads(_dsml_arg_converter(raw, partial=False)) == { + "a": "x", + "b": "42", + } + + def test_missing_string_attribute_partial_value(self): + raw = _param("a", "true", "x") + '<|DSML|parameter name="n">12' + assert json.loads(_dsml_arg_converter(raw, partial=True)) == { + "a": "x", + "n": 12, + } + def test_invalid_json_fallback(self): raw = self._raw(("data", "false", "[broken")) result = json.loads(_dsml_arg_converter(raw, partial=False)) @@ -232,6 +261,47 @@ class TestImplicitParameterClose: } +class TestMissingStringAttribute: + def test_non_streaming_keeps_parameter(self, mock_tokenizer, mock_request): + text = ( + f"{DSML_TOOL_START}" + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n" + f'<|DSML|parameter name="location">Paris{_PARAM_CLOSE}\n' + f"{_param('date', 'true', 'tomorrow')}\n" + f"{DSML_INVOKE_END}{DSML_TOOL_END}" + ) + + result = DeepSeekV4Parser(mock_tokenizer).extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert json.loads(result.tool_calls[0].function.arguments) == { + "location": "Paris", + "date": "tomorrow", + } + + def test_streaming_keeps_parameter(self, mock_tokenizer, mock_request): + chunks = [ + DSML_TOOL_START, + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n", + '<|DSML|parameter name="loc', + 'ation">Par', + "is", + _PARAM_CLOSE, + f"\n{_param('date', 'true', 'tomorrow')}\n", + DSML_INVOKE_END, + DSML_TOOL_END, + ] + + results = simulate_tool_streaming( + DeepSeekV4Parser(mock_tokenizer), mock_request, chunks + ) + + assert json.loads(collect_tool_arguments(results)) == { + "location": "Paris", + "date": "tomorrow", + } + + # ── Bare absorption and duplicate absorption ───────── diff --git a/tests/parser/engine/test_deepseek_v41.py b/tests/parser/engine/test_deepseek_v41.py index cc7d6e0aed..d99ca6b2a1 100644 --- a/tests/parser/engine/test_deepseek_v41.py +++ b/tests/parser/engine/test_deepseek_v41.py @@ -137,10 +137,12 @@ def test_python_argument_conversion_and_partial_values(): '<|DSML| parameter name="object" string="false">' '{"a": [true, null]}' '<|DSML| parameter name="bad" string="false">[broken' + '<|DSML| parameter name="bare">7' '<|DSML| parameter name="text" string="true"> a", ) -_ESCAPED_DSML = re.escape(_DSML) -_PARAM_RE = re.compile( - rf'<{_ESCAPED_DSML}parameter\s+name="([^"]+)"\s+string="(true|false)">' - rf"(.*?)" - rf"(?:|(?=<{_ESCAPED_DSML}parameter\s+name=))", - re.DOTALL, -) -_PARTIAL_PARAM_RE = re.compile( - rf'<{_ESCAPED_DSML}parameter\s+name="([^"]+)"\s+string="(true|false)">' - rf"(.*)$", - re.DOTALL, -) + +def _param_patterns( + param_start: str, param_close: str +) -> tuple[re.Pattern, re.Pattern]: + """Complete and trailing-partial parameter regexes for one DSML dialect. + + The ``string`` attribute is optional: the model sometimes omits it, and + dropping such a parameter hands the client a tool call with no arguments. + """ + start, close = re.escape(param_start), re.escape(param_close) + head = rf'{start}\s+name="([^"]+)"(?:\s+string="([^"]*)")?\s*>' + return ( + re.compile(rf"{head}(.*?)(?:{close}|(?={start}\s+name=))", re.DOTALL), + re.compile(rf"{head}(.*)$", re.DOTALL), + ) + + +_PARAM_RE, _PARTIAL_PARAM_RE = _param_patterns(DSML_PARAM_START, DSML_PARAM_CLOSE) def _dsml_arg_converter( @@ -80,6 +86,8 @@ def _dsml_arg_converter( params: dict[str, object] = {} last_end = 0 + # A missing ``string`` attribute is treated like ``string="false"``: try + # JSON, fall back to the literal text. for m in param_re.finditer(raw_args): name, is_str, value = m.group(1), m.group(2), m.group(3) if is_str == "true": diff --git a/vllm/parser/deepseek_v41.py b/vllm/parser/deepseek_v41.py index 668bdd4a6b..8421600510 100644 --- a/vllm/parser/deepseek_v41.py +++ b/vllm/parser/deepseek_v41.py @@ -5,11 +5,10 @@ import functools from dataclasses import replace -import regex as re - from vllm.parser.deepseek_v4 import ( DeepSeekV4Parser, _dsml_arg_converter, + _param_patterns, deepseek_v4_config, ) from vllm.parser.engine.parser_engine_config import ParserEngineConfig @@ -21,17 +20,7 @@ DSML_INVOKE_END = "" DSML_PARAM_START = "<|DSML| parameter" DSML_PARAM_CLOSE = "" -_PARAM_RE = re.compile( - r'<|DSML| parameter\s+name="([^"]+)"\s+string="(true|false)">' - r"(.*?)" - r"(?:|(?=<|DSML| parameter\s+name=))", - re.DOTALL, -) -_PARTIAL_PARAM_RE = re.compile( - r'<|DSML| parameter\s+name="([^"]+)"\s+string="(true|false)">' - r"(.*)$", - re.DOTALL, -) +_PARAM_RE, _PARTIAL_PARAM_RE = _param_patterns(DSML_PARAM_START, DSML_PARAM_CLOSE) @functools.cache From 42a33039eb43c2adc885329a1c26ee1a5d8cc0fb Mon Sep 17 00:00:00 2001 From: Sun <2114377220@qq.com> Date: Fri, 18 Sep 2026 14:18:41 +0800 Subject: [PATCH 080/143] [Perf] Fuse CohereASR relative attention score accumulation (#55190) Signed-off-by: levius <2114377220@qq.com> Co-authored-by: Codex --- tests/models/multimodal/test_cohere_asr.py | 58 +++++++++++++++++++++- vllm/model_executor/models/cohere_asr.py | 16 ++++-- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/tests/models/multimodal/test_cohere_asr.py b/tests/models/multimodal/test_cohere_asr.py index 85a9b59dd5..950542f900 100644 --- a/tests/models/multimodal/test_cohere_asr.py +++ b/tests/models/multimodal/test_cohere_asr.py @@ -4,9 +4,65 @@ from types import SimpleNamespace import pytest +import torch from vllm.config import SpeechToTextConfig -from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGeneration +from vllm.model_executor.models.cohere_asr import ( + CohereAsrForConditionalGeneration, + RelPositionMultiHeadAttention, +) + + +class _ReferenceRelPositionMultiHeadAttention(RelPositionMultiHeadAttention): + """Relative attention implementation before score accumulation is fused.""" + + def forward(self, query, key, value, mask, pos_emb=None): + q, k, v = self.forward_qkv(query, key, value) + q = q.transpose(1, 2) + + assert pos_emb is not None + n_batch_pos = pos_emb.size(0) + p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) + p = p.transpose(1, 2) + + q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) + q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) + + matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) + matrix_bd = self.rel_shift(matrix_bd) + matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) + matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)] + scores = (matrix_ac + matrix_bd) / self.s_d_k + return self.forward_attention(v, scores, mask) + + +@pytest.mark.parametrize(("query_len", "key_len"), [(2, 2), (3, 5)]) +def test_relative_attention_fuses_score_accumulation(monkeypatch, query_len, key_len): + torch.manual_seed(7) + batch, heads, model_dim = 2, 4, 32 + optimized = RelPositionMultiHeadAttention(heads, model_dim, None, None) + reference = _ReferenceRelPositionMultiHeadAttention(heads, model_dim, None, None) + reference.load_state_dict(optimized.state_dict()) + + query = torch.randn(batch, query_len, model_dim) + key = torch.randn(batch, key_len, model_dim) + value = torch.randn(batch, key_len, model_dim) + pos_emb = torch.randn(1, query_len + key_len - 1, model_dim) + expected = reference(query, key, value, None, pos_emb) + + original_baddbmm = torch.baddbmm + calls = 0 + + def tracked_baddbmm(*args, **kwargs): + nonlocal calls + calls += 1 + return original_baddbmm(*args, **kwargs) + + monkeypatch.setattr(torch, "baddbmm", tracked_baddbmm) + actual = optimized(query, key, value, None, pos_emb) + + torch.testing.assert_close(actual, expected) + assert calls == 1 @pytest.mark.parametrize( diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index 844e8c5e0e..33b4aa7b0d 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -1206,10 +1206,18 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention): matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) matrix_bd = self.rel_shift(matrix_bd) - # drops extra elements in the matrix_bd to match the matrix_ac's size - matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) - matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)] - scores = (matrix_ac + matrix_bd) / self.s_d_k # (batch, head, time1, time2) + # drops extra elements in matrix_bd to match the key sequence length + matrix_bd = matrix_bd[:, :, :, : k.size(-2)] + batch, heads, query_len, key_len = matrix_bd.shape + scale = 1.0 / self.s_d_k + scores = torch.baddbmm( + matrix_bd.flatten(0, 1), + q_with_bias_u.flatten(0, 1), + k.transpose(-2, -1).flatten(0, 1), + beta=scale, + alpha=scale, + ).view(batch, heads, query_len, key_len) + return self.forward_attention(v, scores, mask) From d5f0a6e829faa69d1db289bf62b14dae136c02b2 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Fri, 18 Sep 2026 01:33:41 -0500 Subject: [PATCH 081/143] [Bugfix][DBO] Fix DeepEP low-latency profiling crash with DP+EP+DBO (#57502) Signed-off-by: Micah Williamson Co-authored-by: Andreas Karatzas --- vllm/v1/worker/gpu_model_runner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index c943f02212..cedf95e319 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -6164,6 +6164,11 @@ class GPUModelRunner( num_tokens_padded, None, False ) + num_tokens_unpadded = num_tokens_padded if is_profile else 0 + is_padding = self._prepare_padding_mask( + num_tokens_unpadded, num_tokens_padded + ) + if ubatch_slices_padded is not None: # Adjust values to reflect a single ubatch. # TODO(sage,lucas): this is cruft that should be addressed in @@ -6172,8 +6177,6 @@ class GPUModelRunner( if num_tokens_across_dp is not None: num_tokens_across_dp[:] = num_tokens_padded - is_padding = self._prepare_padding_mask(0, num_tokens_padded) - with ( self.maybe_randomize_inputs( input_ids, inputs_embeds, randomize_inputs=randomize_inputs From 95d723700b1df07b20263c696c0342b92c188bfe Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 18 Sep 2026 01:26:17 -0700 Subject: [PATCH 082/143] [Rust Frontend] Return sampling masks over gRPC (#56777) Co-authored-by: Bugen Zhao Signed-off-by: Biswa Panda --- rust/Cargo.lock | 2 +- rust/proto/Cargo.toml | 2 +- rust/proto/inference.proto | 2 + rust/src/chat/src/output/default/unified.rs | 1 + rust/src/chat/src/output/harmony/tests.rs | 1 + rust/src/chat/tests/roundtrip.rs | 2 + .../src/protocol/logprobs.rs | 2 +- .../src/protocol/logprobs/array.rs | 2 +- .../engine-core-client/src/protocol/mod.rs | 1 + .../engine-core-client/src/protocol/output.rs | 6 +- .../src/protocol/sampling_mask.rs | 237 ++++++++++++++++++ .../engine-core-client/src/tests/client.rs | 12 +- .../src/tests/python_compat.py | 8 +- rust/src/llm/src/error.rs | 8 + rust/src/llm/src/output.rs | 31 +++ rust/src/llm/tests/generate.rs | 69 ++++- rust/src/server/Cargo.toml | 1 + rust/src/server/src/grpc/convert.rs | 44 ++++ rust/src/server/src/grpc/inference.rs | 1 + .../server/src/routes/inference/generate.rs | 6 + .../server/src/routes/openai/completions.rs | 6 + rust/src/text/src/output/decoded.rs | 146 +++++++++++ rust/src/text/src/output/mod.rs | 7 + 23 files changed, 570 insertions(+), 27 deletions(-) create mode 100644 rust/src/engine-core-client/src/protocol/sampling_mask.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 601e11c054..dee3f30789 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5941,7 +5941,7 @@ dependencies = [ [[package]] name = "vllm-proto" -version = "0.3.0" +version = "0.4.0" dependencies = [ "prost", "prost-types", diff --git a/rust/proto/Cargo.toml b/rust/proto/Cargo.toml index cac268bc74..492cb5a06d 100644 --- a/rust/proto/Cargo.toml +++ b/rust/proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vllm-proto" -version = "0.3.0" +version = "0.4.0" edition.workspace = true license.workspace = true description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend" diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index 46313f915a..4eaf6f92f4 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -160,6 +160,8 @@ message SequenceOutput { // Only present in final output for this sequence optional FinishInfo finish_info = 8; + // Only present in final output when sampling-mask capture is enabled. + repeated TokenIds sampling_mask = 10; } // Prompt info, returned in the first response diff --git a/rust/src/chat/src/output/default/unified.rs b/rust/src/chat/src/output/default/unified.rs index 964f0bf35a..023e3c3ffa 100644 --- a/rust/src/chat/src/output/default/unified.rs +++ b/rust/src/chat/src/output/default/unified.rs @@ -438,6 +438,7 @@ mod tests { finish_reason: crate::FinishReason::Stop(None), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, })), } } diff --git a/rust/src/chat/src/output/harmony/tests.rs b/rust/src/chat/src/output/harmony/tests.rs index 8dcf84dd98..035aef3cd4 100644 --- a/rust/src/chat/src/output/harmony/tests.rs +++ b/rust/src/chat/src/output/harmony/tests.rs @@ -58,6 +58,7 @@ fn finished() -> Finished { finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, } } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index db3559c44f..9cba9f0aa2 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -806,6 +806,7 @@ fn decoded_completion_stream( finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, })), } }); @@ -817,6 +818,7 @@ fn decoded_completion_stream( finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }); events.push(DecodedTextEvent::TextDelta { decoded: vllm_text::DecodedText::unattributed(chunk.delta), diff --git a/rust/src/engine-core-client/src/protocol/logprobs.rs b/rust/src/engine-core-client/src/protocol/logprobs.rs index 646c106a04..72d852ba2c 100644 --- a/rust/src/engine-core-client/src/protocol/logprobs.rs +++ b/rust/src/engine-core-client/src/protocol/logprobs.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project -mod array; +pub(crate) mod array; #[cfg(test)] mod tests; mod wire; diff --git a/rust/src/engine-core-client/src/protocol/logprobs/array.rs b/rust/src/engine-core-client/src/protocol/logprobs/array.rs index 4adf1ae975..919ef80f46 100644 --- a/rust/src/engine-core-client/src/protocol/logprobs/array.rs +++ b/rust/src/engine-core-client/src/protocol/logprobs/array.rs @@ -50,7 +50,7 @@ pub(super) fn decode_array2_u32( }) } -pub(super) fn decode_array1_u32( +pub(crate) fn decode_array1_u32( value: WireNdArray, field: &str, frames: &[Bytes], diff --git a/rust/src/engine-core-client/src/protocol/mod.rs b/rust/src/engine-core-client/src/protocol/mod.rs index 8f4ee06127..742fed59bd 100644 --- a/rust/src/engine-core-client/src/protocol/mod.rs +++ b/rust/src/engine-core-client/src/protocol/mod.rs @@ -30,6 +30,7 @@ pub mod multimodal; pub mod output; pub mod request; pub mod sampling; +pub mod sampling_mask; mod serde_utils; pub mod stats; pub mod structured_outputs; diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index c8a8705435..a45cbe3731 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -15,6 +15,7 @@ use super::serde_utils::AllowTrailingFields; use super::utility::UtilityOutput; use crate::error::{Error, Result, ext_value_decode}; use crate::protocol::logprobs::MaybeWireLogprobs; +use crate::protocol::sampling_mask::MaybeWireSamplingMask; use crate::protocol::stats::{PrefillStats, SchedulerStats}; use crate::protocol::{OpaqueValue, decode_msgpack}; @@ -127,7 +128,7 @@ pub struct EngineCoreOutput { #[serde(default)] pub mm_cache_miss_hashes: Option>, #[serde(default)] - pub new_sampling_mask: Option, + pub new_sampling_mask: Option, /// Per-request speculative-decoding acceptance metrics, set on the final /// output when `--per-request-spec-decode-metrics` is enabled. Opaque here; /// the Rust frontend does not yet surface it in responses. @@ -150,6 +151,9 @@ impl EngineCoreOutput { self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take()) .map(|value| value.resolve(frames, "new_prompt_logprobs_tensors")) .transpose()?; + self.new_sampling_mask = (self.new_sampling_mask.take()) + .map(|value| value.resolve(frames, "new_sampling_mask")) + .transpose()?; Ok(()) } } diff --git a/rust/src/engine-core-client/src/protocol/sampling_mask.rs b/rust/src/engine-core-client/src/protocol/sampling_mask.rs new file mode 100644 index 0000000000..7f5c4592a7 --- /dev/null +++ b/rust/src/engine-core-client/src/protocol/sampling_mask.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::ops::{Deref, DerefMut}; + +use bytes::Bytes; +use enum_as_inner::EnumAsInner; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_tuple::{Deserialize_tuple, Serialize_tuple}; + +use crate::error::{Error, Result, bail_ext_value_decode}; +use crate::protocol::logprobs::array::decode_array1_u32; +use crate::protocol::tensor::WireNdArray; + +/// Decoded sampling-mask support sets, one row per generated token position. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SamplingMask { + pub rows: Vec>, +} + +/// Python `SamplingMaskLists` tuple before ndarray raw views are resolved. +#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)] +pub struct WireSamplingMask { + token_ids: WireNdArray, + #[serde(default)] + offsets: Option, + #[serde(default)] + cu_num_generated_tokens: Option>, +} + +/// Sampling-mask field while it transitions from Python wire data to rows. +#[derive(Debug, Clone, PartialEq, EnumAsInner)] +pub enum MaybeWireSamplingMask { + Wire(Box), + Direct(SamplingMask), +} + +impl Deref for MaybeWireSamplingMask { + type Target = SamplingMask; + + fn deref(&self) -> &Self::Target { + match self { + Self::Wire(_) => panic!("sampling mask is still in wire format"), + Self::Direct(value) => value, + } + } +} + +impl DerefMut for MaybeWireSamplingMask { + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + Self::Wire(_) => panic!("sampling mask is still in wire format"), + Self::Direct(value) => value, + } + } +} + +impl<'de> Deserialize<'de> for MaybeWireSamplingMask { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + WireSamplingMask::deserialize(deserializer).map(|value| Self::Wire(Box::new(value))) + } +} + +impl Serialize for MaybeWireSamplingMask { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + match self { + Self::Wire(value) => value.serialize(serializer), + Self::Direct(value) => WireSamplingMask::from_direct(value) + .map_err(serde::ser::Error::custom)? + .serialize(serializer), + } + } +} + +impl MaybeWireSamplingMask { + pub(super) fn resolve(self, frames: &[Bytes], field_prefix: &str) -> Result { + match self { + Self::Direct(value) => Ok(Self::Direct(value)), + Self::Wire(value) => value.resolve(frames, field_prefix).map(Self::Direct), + } + } +} + +impl WireSamplingMask { + fn from_direct(value: &SamplingMask) -> std::result::Result { + let [row] = value.rows.as_slice() else { + return Err(format!( + "per-request engine-core sampling mask must contain exactly one row, got {}", + value.rows.len() + )); + }; + let token_ids = row.iter().map(|&token_id| i64::from(token_id)).collect::>(); + Ok(Self { + token_ids: WireNdArray::from_i64(vec![token_ids.len()], token_ids)?, + offsets: None, + cu_num_generated_tokens: None, + }) + } + + fn resolve(self, frames: &[Bytes], field_prefix: &str) -> Result { + if let Some(indices) = self.cu_num_generated_tokens { + bail_ext_value_decode!( + "{field_prefix}.cu_num_generated_tokens: expected None for per-request engine-core sampling-mask payload, got {indices:?}" + ); + } + + let token_ids = + decode_array1_u32(self.token_ids, &format!("{field_prefix}.token_ids"), frames)?; + let Some(offsets) = self.offsets else { + return Ok(SamplingMask { + rows: vec![token_ids], + }); + }; + let offsets = decode_array1_u32(offsets, &format!("{field_prefix}.offsets"), frames)?; + if offsets.first().copied() != Some(0) { + bail_ext_value_decode!("{field_prefix}.offsets: first offset must be zero"); + } + if offsets.windows(2).any(|pair| pair[0] > pair[1]) { + bail_ext_value_decode!("{field_prefix}.offsets: offsets must be nondecreasing"); + } + if offsets.last().copied().map(|value| value as usize) != Some(token_ids.len()) { + bail_ext_value_decode!( + "{field_prefix}.offsets: final offset {:?} does not match token_ids length {}", + offsets.last(), + token_ids.len() + ); + } + + let rows = offsets + .windows(2) + .map(|pair| token_ids[pair[0] as usize..pair[1] as usize].to_vec()) + .collect(); + Ok(SamplingMask { rows }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::dtype::{NumpyDtype, TensorDtype}; + use crate::protocol::output::{ + EngineCoreOutput, RequestBatchOutputs, decode_engine_core_outputs, + }; + use crate::protocol::tensor::WireArrayData; + use crate::protocol::{decode_msgpack, encode_msgpack}; + + #[test] + fn decodes_single_position_without_offsets() { + let wire = WireSamplingMask { + token_ids: WireNdArray::from_i64(vec![3], vec![2, 12, 16]).unwrap(), + offsets: None, + cu_num_generated_tokens: None, + }; + let encoded = encode_msgpack(&wire).unwrap(); + let decoded: MaybeWireSamplingMask = decode_msgpack(&encoded).unwrap(); + let decoded = decoded.resolve(&[], "new_sampling_mask").unwrap(); + + assert_eq!(decoded.into_direct().unwrap().rows, vec![vec![2, 12, 16]]); + } + + #[test] + fn decodes_csr_rows_and_rejects_bad_terminal_offset() { + let wire = WireSamplingMask { + token_ids: WireNdArray::from_i64(vec![4], vec![2, 12, 16, 18]).unwrap(), + offsets: Some(WireNdArray::from_i64(vec![3], vec![0, 2, 4]).unwrap()), + cu_num_generated_tokens: None, + }; + let decoded = MaybeWireSamplingMask::Wire(Box::new(wire)) + .resolve(&[], "new_sampling_mask") + .unwrap(); + assert_eq!( + decoded.into_direct().unwrap().rows, + vec![vec![2, 12], vec![16, 18]] + ); + + let malformed = WireSamplingMask { + token_ids: WireNdArray::from_i64(vec![2], vec![2, 12]).unwrap(), + offsets: Some(WireNdArray::from_i64(vec![2], vec![0, 1]).unwrap()), + cu_num_generated_tokens: None, + }; + let error = MaybeWireSamplingMask::Wire(Box::new(malformed)) + .resolve(&[], "new_sampling_mask") + .unwrap_err(); + assert!(error.to_string().contains("final offset")); + } + + #[test] + fn decodes_sampling_mask_from_multipart_aux_frames() { + let output = EngineCoreOutput { + request_id: "req-multipart-mask".to_string(), + new_token_ids: vec![16, 18], + new_sampling_mask: Some(MaybeWireSamplingMask::Wire(Box::new(WireSamplingMask { + token_ids: WireNdArray { + dtype: NumpyDtype::little(TensorDtype::I32), + shape: vec![4], + data: WireArrayData::AuxIndex(1), + }, + offsets: Some(WireNdArray { + dtype: NumpyDtype::little(TensorDtype::I64), + shape: vec![3], + data: WireArrayData::AuxIndex(2), + }), + cu_num_generated_tokens: None, + }))), + ..Default::default() + }; + let primary = encode_msgpack(&crate::protocol::output::EngineCoreOutputs::from( + RequestBatchOutputs { + outputs: vec![output], + ..Default::default() + }, + )) + .unwrap(); + let token_ids = + [2_i32, 12, 16, 18].into_iter().flat_map(i32::to_le_bytes).collect::>(); + let offsets = [0_i64, 2, 4].into_iter().flat_map(i64::to_le_bytes).collect::>(); + + let decoded = decode_engine_core_outputs(&[ + Bytes::from(primary), + Bytes::from(token_ids), + Bytes::from(offsets), + ]) + .unwrap(); + let mask = decoded.as_request_batch().unwrap().outputs[0] + .new_sampling_mask + .as_ref() + .unwrap(); + + assert_eq!(mask.rows, vec![vec![2, 12], vec![16, 18]]); + } +} diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 739c34d6b7..eb8e713fb4 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2712,15 +2712,15 @@ fn python_msgpack_fixtures_match_rust_encoding() { decode_value(&rmp_serde::to_vec_named(&expected_multimodal_request.mm_features).unwrap()); assert_eq!(python_mm_features, rust_mm_features); - let decoded_sampling_mask_outputs: EngineCoreOutputs = - rmp_serde::from_slice(&sampling_mask_outputs_bytes).unwrap(); + let decoded_sampling_mask_outputs = + decode_engine_core_outputs(&[bytes::Bytes::from(sampling_mask_outputs_bytes)]).unwrap(); let sampling_mask_output = &decoded_sampling_mask_outputs.as_request_batch().unwrap().outputs[0]; assert!(sampling_mask_output.mm_cache_miss_hashes.is_none()); - assert!(matches!( - sampling_mask_output.new_sampling_mask.as_ref(), - Some(Value::Array(fields)) if fields.len() == 3 - )); + assert_eq!( + sampling_mask_output.new_sampling_mask.as_ref().unwrap().rows, + vec![vec![2, 12, 16, 17, 18]] + ); let decoded_outputs: EngineCoreOutputs = rmp_serde::from_slice(&outputs_bytes).unwrap(); // Match msgspec's base-schema result for the same extended Python message. diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index 5013f9e2a5..a19dc9fdaa 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -234,14 +234,10 @@ sampling_mask_wire = [ [ ", + /// Sampling support sets aligned one-to-one with generated token positions. + pub sampling_mask: Option, } /// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for @@ -155,6 +158,8 @@ pub struct GenerateOutput { /// Connector-specific encoder cache transfer parameters for disaggregated /// serving. pub ec_transfer_params: Option, + /// Sampling support sets aligned one-to-one with `token_ids`. + pub sampling_mask: Option, } impl GenerateOutput { @@ -202,6 +207,7 @@ impl GenerateOutput { cached_token_count: 0, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, } } } @@ -271,6 +277,16 @@ impl Stream for GenerateOutputStream { } let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap()); + let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap()); + if let Some(mask) = sampling_mask.as_ref() + && mask.rows.len() != raw.new_token_ids.len() + { + return Poll::Ready(Some(Err(crate::Error::SamplingMaskTokenCountMismatch { + request_id: raw.request_id, + token_count: raw.new_token_ids.len(), + row_count: mask.rows.len(), + }))); + } let cached_token_count = raw .prefill_stats .as_ref() @@ -291,6 +307,7 @@ impl Stream for GenerateOutputStream { cached_token_count, kv_transfer_params: raw.kv_transfer_params, ec_transfer_params: raw.ec_transfer_params, + sampling_mask, }; Poll::Ready(Some(Ok(output))) @@ -341,6 +358,7 @@ impl> + Send> T { while let Some(output) = stream.next().await.transpose()? { cached_token_count = cached_token_count.max(output.cached_token_count); + let sampling_mask = output.sampling_mask; if let Some(info) = output.prompt_info { if prompt_token_ids.is_none() { prompt_token_ids = Some(info.prompt_token_ids.to_vec()); @@ -359,6 +377,9 @@ impl> + Send> T { existing.logprobs = Some(step_logprobs); } } + if let Some(mut mask) = sampling_mask { + existing.sampling_mask.get_or_insert_default().rows.append(&mut mask.rows); + } } else { collected = Some(CollectedGenerateOutput { request_id: output.request_id, @@ -374,6 +395,7 @@ impl> + Send> T { }, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask, }); } @@ -387,6 +409,15 @@ impl> + Send> T { }; collected.kv_transfer_params = output.kv_transfer_params; collected.ec_transfer_params = output.ec_transfer_params; + if let Some(mask) = collected.sampling_mask.as_ref() + && mask.rows.len() != collected.token_ids.len() + { + return Err(crate::Error::SamplingMaskTokenCountMismatch { + request_id: collected.request_id, + token_count: collected.token_ids.len(), + row_count: mask.rows.len(), + }); + } return Ok(collected); } } diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index 2d09aff889..09b5199767 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use std::collections::BTreeSet; -use std::sync::Once; +use std::sync::{Arc, Once}; use std::time::Duration; use futures::StreamExt as _; @@ -18,6 +18,7 @@ use vllm_engine_core_client::protocol::output::{ }; use vllm_engine_core_client::protocol::request::EngineCoreRequest; use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams; +use vllm_engine_core_client::protocol::sampling_mask::{MaybeWireSamplingMask, SamplingMask}; use vllm_engine_core_client::protocol::stats::PrefillStats; use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; @@ -326,6 +327,11 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() { num_local_cached_tokens: 1, ..Default::default() }), + new_sampling_mask: Some(MaybeWireSamplingMask::Direct( + SamplingMask { + rows: vec![vec![1, 33, 99]], + }, + )), ..request_output_with_logprobs( &request.request_id, vec![33], @@ -334,15 +340,22 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() { Some(prompt_logprobs()), ) }, - request_output_with_logprobs_and_kv( - &request.request_id, - vec![44], - Some(EngineCoreFinishReason::Stop), - Some(logprobs_for_position(44, -0.3, 1, 88, -0.4)), - None, - Some(serde_json::json!({"connector": "x"})), - None, - ), + EngineCoreOutput { + new_sampling_mask: Some(MaybeWireSamplingMask::Direct( + SamplingMask { + rows: vec![vec![2, 44, 88]], + }, + )), + ..request_output_with_logprobs_and_kv( + &request.request_id, + vec![44], + Some(EngineCoreFinishReason::Stop), + Some(logprobs_for_position(44, -0.3, 1, 88, -0.4)), + None, + Some(serde_json::json!({"connector": "x"})), + None, + ) + }, ], ..Default::default() } @@ -375,6 +388,42 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() { collected.kv_transfer_params, Some(serde_json::json!({"connector": "x"})) ); + assert_eq!( + collected.sampling_mask, + Some(SamplingMask { + rows: vec![vec![1, 33, 99], vec![2, 44, 88]], + }) + ); +} + +#[tokio::test] +async fn collect_output_rejects_partial_sampling_mask() { + let output = vllm_llm::GenerateOutput { + request_id: "req-partial-mask".to_string(), + prompt_info: Some(GeneratePromptInfo { + prompt_token_ids: Arc::from([11_u32, 22]), + prompt_logprobs: None, + }), + token_ids: vec![33, 44], + logprobs: None, + finish_reason: Some(FinishReason::Length), + cached_token_count: 0, + kv_transfer_params: None, + ec_transfer_params: None, + sampling_mask: Some(SamplingMask { + rows: vec![vec![1, 33, 99]], + }), + }; + + let error = futures::stream::iter([Ok(output)]).collect_output().await.unwrap_err(); + assert!(matches!( + error, + Error::SamplingMaskTokenCountMismatch { + token_count: 2, + row_count: 1, + .. + } + )); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index caccbdefc5..8990ba2f3b 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -19,6 +19,7 @@ itertools.workspace = true libc.workspace = true llm-multimodal.workspace = true openssl.workspace = true +prost.workspace = true prost-types.workspace = true rmp-serde.workspace = true rmpv.workspace = true diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index 5f6ce5d42b..0f28e147ef 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -316,6 +316,11 @@ pub fn to_sequence_output( _ => (vec![], vec![], vec![]), }; + let sampling_mask = finished + .and_then(|finished| finished.sampling_mask.as_ref()) + .map(|mask| mask.rows.iter().map(|row| pb::TokenIds { ids: row.clone() }).collect()) + .unwrap_or_default(); + Ok(pb::SequenceOutput { index: 0, // TODO: multi-sequence (n > 1) not supported text: if opts.output_text { @@ -333,6 +338,7 @@ pub fn to_sequence_output( ranks: rank_values, candidate_tokens: candidates, finish_info, + sampling_mask, }) } @@ -524,6 +530,7 @@ impl ResponseOpts { mod tests { use prost::Message as _; use vllm_engine_core_client::protocol::output::StopReason; + use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use vllm_text::{ FinishReason, Finished, Prompt, SamplingHints, SamplingLimits, lower_sampling_params, }; @@ -720,6 +727,7 @@ mod tests { finish_reason: reason, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, } } @@ -804,4 +812,40 @@ mod tests { assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32); assert_eq!(finish.stop_reason, Some(PbStopReason::EosTokenId(30))); } + + #[test] + fn sequence_output_only_carries_sampling_mask_on_terminal_output() { + let mut fin = finished(FinishReason::Length); + fin.sampling_mask = Some(SamplingMask { + rows: vec![vec![1, 10], vec![2, 20]], + }); + + let terminal = + to_sequence_output("", &[10, 20], None, Some(&fin), &ResponseOpts::default()) + .expect("terminal output"); + let intermediate = to_sequence_output("", &[9], None, None, &ResponseOpts::default()) + .expect("intermediate output"); + + assert_eq!( + terminal.sampling_mask, + vec![ + pb::TokenIds { ids: vec![1, 10] }, + pb::TokenIds { ids: vec![2, 20] } + ] + ); + assert!(intermediate.sampling_mask.is_empty()); + } + + #[test] + fn sampling_mask_uses_additive_proto_tag_ten() { + let response = pb::SequenceOutput { + sampling_mask: vec![pb::TokenIds { ids: vec![7, 8] }], + ..Default::default() + }; + + assert_eq!( + response.encode_to_vec(), + vec![0x52, 0x04, 0x0a, 0x02, 0x07, 0x08] + ); + } } diff --git a/rust/src/server/src/grpc/inference.rs b/rust/src/server/src/grpc/inference.rs index 72943b6af0..d56415c747 100644 --- a/rust/src/server/src/grpc/inference.rs +++ b/rust/src/server/src/grpc/inference.rs @@ -257,6 +257,7 @@ impl pb::inference_server::Inference for InferenceServiceImpl { finish_reason: collected.finish_reason, kv_transfer_params: collected.kv_transfer_params, ec_transfer_params: collected.ec_transfer_params, + sampling_mask: collected.sampling_mask, }; let outputs = convert::to_sequence_output( diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index dfb7e6cbb1..f82059dffd 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -481,6 +481,7 @@ mod tests { cached_token_count: 0, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), Ok(GenerateOutput { request_id: String::new(), @@ -494,6 +495,7 @@ mod tests { cached_token_count: 2, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), ]); @@ -575,6 +577,7 @@ mod tests { cached_token_count: 0, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, } } @@ -658,6 +661,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, prompt_token_ids: vec![10, 20], + sampling_mask: None, }; let response = collect_generate( @@ -744,6 +748,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, prompt_token_ids: vec![10, 20], + sampling_mask: None, }; let response = collect_generate( @@ -776,6 +781,7 @@ mod tests { }, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, prompt_token_ids, }; diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index a122afbcd9..06874a43bd 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -694,6 +694,7 @@ mod tests { ))), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -806,6 +807,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -858,6 +860,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -913,6 +916,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -985,6 +989,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -1059,6 +1064,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); diff --git a/rust/src/text/src/output/decoded.rs b/rust/src/text/src/output/decoded.rs index a438c8e267..d3dc422656 100644 --- a/rust/src/text/src/output/decoded.rs +++ b/rust/src/text/src/output/decoded.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use tracing::{Level, debug, trace}; use vllm_engine_core_client::AbortCause; use vllm_engine_core_client::protocol::output::StopReason; +use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use vllm_llm::{FinishReason, GenerateOutput, TokenUsage}; use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder}; @@ -50,6 +51,8 @@ pub struct Finished { /// Connector-specific encoder cache transfer parameters for disaggregated /// serving. pub ec_transfer_params: Option, + /// Sampling support sets aligned with all generated token positions. + pub sampling_mask: Option, } /// Sample metadata emitted by one engine output update. @@ -120,6 +123,7 @@ pub async fn decoded_text_event_stream( let mut sampled_token_ids = Vec::new(); let mut output_token_count: usize = 0; let mut sampled_logprobs: Option = None; + let mut sampling_mask: Option = None; while let Some(next) = raw_stream.next().await { let output = next?; @@ -213,6 +217,7 @@ pub async fn decoded_text_event_stream( let mut new_token_ids = output.token_ids; let mut new_logprobs = output.logprobs; + let mut new_sampling_mask = output.sampling_mask; // Trim tokens and logprobs if we matched stop string. if let Some(num_tokens) = truncate_tokens_to { @@ -220,9 +225,15 @@ pub async fn decoded_text_event_stream( if let Some(logprobs) = &mut new_logprobs { logprobs.positions.truncate(num_tokens); } + if let Some(mask) = &mut new_sampling_mask { + mask.rows.truncate(num_tokens); + } } output_token_count += new_token_ids.len(); + if let Some(mut mask) = new_sampling_mask { + sampling_mask.get_or_insert_default().rows.append(&mut mask.rows); + } let decoded_logprobs = new_logprobs .as_ref() @@ -246,6 +257,16 @@ pub async fn decoded_text_event_stream( } if let Some(reason) = finish_reason { + if let Some(mask) = sampling_mask.as_ref() + && mask.rows.len() != output_token_count + { + return Err(vllm_llm::Error::SamplingMaskTokenCountMismatch { + request_id, + token_count: output_token_count, + row_count: mask.rows.len(), + } + .into()); + } // Flush any remaining buffered text. let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?; let text_len = full_decoded.text.len(); @@ -293,6 +314,7 @@ pub async fn decoded_text_event_stream( finish_reason: reason, kv_transfer_params, ec_transfer_params, + sampling_mask, })), }) .await; @@ -354,6 +376,7 @@ mod tests { use futures::{Stream, StreamExt as _, stream}; use vllm_engine_core_client::AbortCause; + use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob}; use vllm_llm::GenerateOutput; use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer}; @@ -384,6 +407,73 @@ mod tests { s.bytes().map(u32::from).collect() } + #[tokio::test] + async fn sampling_masks_survive_text_decode_and_collection() { + let prompt: Arc<[u32]> = Arc::from([]); + let mut first = GenerateOutput::for_test(Some(prompt), vec![b'a' as u32], None); + first.sampling_mask = Some(SamplingMask { + rows: vec![vec![1, b'a' as u32]], + }); + let mut terminal = + GenerateOutput::for_test(None, vec![b'b' as u32], Some(FinishReason::Length)); + terminal.sampling_mask = Some(SamplingMask { + rows: vec![vec![2, b'b' as u32]], + }); + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); + + let collected = decoded_text_event_stream( + "test-mask".into(), + tokenizer, + stream::iter([Ok(first), Ok(terminal)]), + TextDecodeOptions::default(), + false, + ) + .collect_output() + .await + .unwrap(); + + assert_eq!( + collected.sampling_mask, + Some(SamplingMask { + rows: vec![vec![1, b'a' as u32], vec![2, b'b' as u32]], + }) + ); + } + + #[tokio::test] + async fn sampling_mask_mismatch_reuses_llm_error() { + let prompt: Arc<[u32]> = Arc::from([]); + let mut terminal = GenerateOutput::for_test( + Some(prompt), + vec![b'a' as u32, b'b' as u32], + Some(FinishReason::Length), + ); + terminal.sampling_mask = Some(SamplingMask { + rows: vec![vec![1, b'a' as u32]], + }); + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); + + let error = decoded_text_event_stream( + "test-mask-mismatch".into(), + tokenizer, + stream::iter([Ok(terminal)]), + TextDecodeOptions::default(), + false, + ) + .collect_output() + .await + .unwrap_err(); + + assert!(matches!( + error, + crate::Error::Llm(vllm_llm::Error::SamplingMaskTokenCountMismatch { + token_count: 2, + row_count: 1, + .. + }) + )); + } + fn opts(stop: &[&str], min_tokens: u32) -> TextDecodeOptions { TextDecodeOptions { stop_strings: Some(stop.iter().map(|s| s.to_string()).collect()), @@ -472,6 +562,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), ), ], @@ -503,6 +594,7 @@ mod tests { ))), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), ), ], @@ -576,6 +668,60 @@ mod tests { assert!(output.finish_reason.is_stop()); } + #[tokio::test] + async fn stream_stop_string_mid_chunk_keeps_token_logprob_mask_alignment() { + let token_ids = ascii_tokens("abcxyz"); + let mut raw = GenerateOutput::for_test( + Some(Arc::from([])), + token_ids.clone(), + Some(FinishReason::Length), + ); + raw.logprobs = Some(Logprobs { + positions: token_ids + .iter() + .map(|&token_id| PositionLogprobs { + entries: vec![TokenLogprob { + token_id, + logprob: -0.25, + rank: 1, + }], + }) + .collect(), + }); + raw.sampling_mask = Some(SamplingMask { + rows: token_ids.iter().map(|&token_id| vec![token_id, token_id + 100]).collect(), + }); + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); + + let output = decoded_text_event_stream( + "test-mid-chunk-stop".into(), + tokenizer, + stream::iter([Ok(raw)]), + opts(&["cx"], 0), + false, + ) + .collect_output() + .await + .unwrap(); + + assert_eq!(output.text, "ab"); + assert!(output.finish_reason.is_stop()); + assert_eq!(output.token_ids, ascii_tokens("abcx")); + assert_eq!(output.logprobs.as_ref().unwrap().positions.len(), 4); + assert_eq!( + output.sampling_mask, + Some(SamplingMask { + rows: vec![ + vec![b'a' as u32, b'a' as u32 + 100], + vec![b'b' as u32, b'b' as u32 + 100], + vec![b'c' as u32, b'c' as u32 + 100], + vec![b'x' as u32, b'x' as u32 + 100], + ], + }) + ); + assert_eq!(output.usage.output_token_count, 4); + } + #[tokio::test] async fn stream_stop_string_at_end() { let output = run_to_completion(ascii_tokens("abcxyz"), opts(&["xyz"], 0)).await; diff --git a/rust/src/text/src/output/mod.rs b/rust/src/text/src/output/mod.rs index a415bb8900..1293ac6025 100644 --- a/rust/src/text/src/output/mod.rs +++ b/rust/src/text/src/output/mod.rs @@ -17,6 +17,7 @@ mod logprobs; use std::sync::Arc; use futures::{StreamExt as _, pin_mut}; +use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use crate::{Error, FinishReason, Result, TextOutputStream}; @@ -35,6 +36,8 @@ pub struct CollectedTextOutput { /// Connector-specific encoder cache transfer parameters for disaggregated /// serving. pub ec_transfer_params: Option, + /// Sampling support sets aligned with generated token positions. + pub sampling_mask: Option, } #[allow(clippy::manual_async_fn, reason = "specify `Send` bound")] @@ -90,6 +93,7 @@ impl T { usage: vllm_llm::TokenUsage::default(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }) }; @@ -99,6 +103,7 @@ impl T { collected.usage = finished.usage; collected.kv_transfer_params = finished.kv_transfer_params; collected.ec_transfer_params = finished.ec_transfer_params; + collected.sampling_mask = finished.sampling_mask; return Ok(collected); } } @@ -173,6 +178,7 @@ mod tests { finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, })), }), ]); @@ -295,6 +301,7 @@ mod tests { finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, })), }), ]); From 8da75d61904cd0b2c5c322746ea15251773ec51f Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Fri, 18 Sep 2026 16:39:27 +0800 Subject: [PATCH 083/143] C3x SM100 FP8 blockwise - Pad activation scales to multiple of 4 (#57377) Signed-off-by: Thien Tran Co-authored-by: Codex --- .buildkite/test_areas/kernels.yaml | 3 ++ ...scaled_mm_blockwise_sm100_fp8_dispatch.cuh | 33 ++++++++++++++++--- tests/kernels/quantization/test_block_fp8.py | 9 +++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 583378fdb4..207d827f36 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -367,6 +367,7 @@ steps: - csrc/quantization/fp4/ - csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu - csrc/libtorch_stable/quantization/fp4/ + - csrc/libtorch_stable/quantization/w8a8/cutlass/ - csrc/attention/mla/ - csrc/quantization/cutlass_w8a8/moe/ - vllm/model_executor/layers/fusion/ @@ -408,6 +409,7 @@ steps: - tests/kernels/test_ll_bf16_gemm.py - tests/kernels/test_top_k_per_row.py - tests/kernels/quantization/test_silu_mul_nvfp4_quant.py + - tests/kernels/quantization/test_block_fp8.py - tests/models/quantization/test_nvfp4.py commands: - nvidia-smi @@ -442,6 +444,7 @@ steps: pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2' pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8' + pytest -v -s tests/kernels/quantization/test_block_fp8.py::test_w8a8_block_fp8_cutlass_matmul pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh index 8f74f29914..fa2add9182 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh @@ -152,6 +152,29 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te int32_t m = a.size(0), n = b.size(1), k = a.size(1); + // The SM100 blockwise SF copy requires the SFA M extent to be a multiple + // of 4 (4-row copy atom). For misaligned M on the non-swapAB path, pad only + // the per-token activation scales (column-major (m, k/128), 1/64 the size + // of the bf16 activation) and build layout_SFA from the padded extent. + int32_t m_sf = m; + std::optional a_scales_pad; + void const* a_scales_data = a_scales.data_ptr(); + if constexpr (!swap_ab) { + if (m % 4 != 0) { + m_sf = (m + 3) & ~3; + int64_t num_groups = a_scales.numel() / m; + a_scales_pad = torch::stable::new_empty( + a_scales, {num_groups * m_sf}, + torch::headeronly::ScalarType::Float); + cudaMemcpy2DAsync(a_scales_pad->data_ptr(), m_sf * sizeof(float), + a_scales.data_ptr(), m * sizeof(float), + m * sizeof(float), num_groups, + cudaMemcpyDeviceToDevice, + get_current_cuda_stream(a.get_device())); + a_scales_data = a_scales_pad->data_ptr(); + } + } + StrideA a_stride; StrideB b_stride; StrideC c_stride; @@ -162,16 +185,16 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te c_stride = cutlass::make_cute_packed_stride(StrideC{}, swap_ab ? cute::make_shape(n, m, 1) : cute::make_shape(m, n, 1)); - LayoutSFA layout_SFA = swap_ab ? + LayoutSFA layout_SFA = swap_ab ? ScaleConfig::tile_atom_to_shape_SFA(make_shape(n, m, k, 1)) : - ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + ScaleConfig::tile_atom_to_shape_SFA(make_shape(m_sf, n, k, 1)); LayoutSFB layout_SFB = swap_ab ? ScaleConfig::tile_atom_to_shape_SFB(make_shape(n, m, k, 1)) : ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); auto a_ptr = static_cast(a.data_ptr()); auto b_ptr = static_cast(b.data_ptr()); - auto a_scales_ptr = static_cast(a_scales.data_ptr()); + auto a_scales_ptr = static_cast(a_scales_data); auto b_scales_ptr = static_cast(b_scales.data_ptr()); typename GemmKernel::MainloopArguments mainloop_args{}; @@ -212,7 +235,9 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::stable::Tensor& out, constexpr int TILE_K = 128; // TODO: better heuristics - bool swap_ab = (m < 16) || (m % 4 != 0); + // swapAB only wins for small M; misaligned M >= 65 is faster on the + // non-swapAB path with padded activation scales (measured on GB300). + bool swap_ab = (m <= 64); bool use_tma_epilogue = (m * n) % 4 == 0; if (!swap_ab) { constexpr int TILE_N = 128; diff --git a/tests/kernels/quantization/test_block_fp8.py b/tests/kernels/quantization/test_block_fp8.py index d02527aba0..038b94bd2d 100644 --- a/tests/kernels/quantization/test_block_fp8.py +++ b/tests/kernels/quantization/test_block_fp8.py @@ -162,11 +162,16 @@ def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed): @pytest.mark.skipif( not current_platform.is_cuda(), reason="CUTLASS only supported on CUDA platform." ) +@pytest.mark.parametrize( + # 65/66/67 cover all M%4 residue classes above the SM100 swapAB + # threshold (m <= 64); 1026 crosses multiple 128-row SF atoms. + "M", + [32, 65, 66, 67, 1026], +) @torch.inference_mode() -def test_w8a8_block_fp8_cutlass_matmul(): +def test_w8a8_block_fp8_cutlass_matmul(M): # Test simple case where weight.shape % 128 != 0, # like in DSV3 kv_a_proj_with_mqa - M = 32 N = 576 K = 7168 block_size = [128, 128] From 32636580a6f1c3bc41deefc4bf7800f850c24034 Mon Sep 17 00:00:00 2001 From: olka-amd Date: Fri, 18 Sep 2026 11:49:53 +0300 Subject: [PATCH 084/143] [Bugfix][MLA] TritonMLA: fix illegal memory access on causal multi-token decode (#51065) Signed-off-by: Oleksandr Kachur Co-authored-by: Claude Opus 5 --- .../test_triton_mla_causal_verify_flatten.py | 215 ++++++++++++++++++ vllm/v1/attention/backends/mla/triton_mla.py | 91 +++----- 2 files changed, 252 insertions(+), 54 deletions(-) create mode 100644 tests/kernels/attention/test_triton_mla_causal_verify_flatten.py diff --git a/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py b/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py new file mode 100644 index 0000000000..a27552c90e --- /dev/null +++ b/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the multi-token decode flatten in ``TritonMLAImpl.forward_mqa``. + +``decode_attention_fwd`` has no causal flag: it launches one program per row of +``q`` and reads that row's KV extent from ``B_Seqlen``. So a ``query_len``-token +decode block is flattened to one row per query token, and intra-block causality +has to be expressed through the per-row sequence lengths -- row ``t`` gets +``seq_len - (query_len - 1) + t``, i.e. the committed prefix plus block tokens +``0..t``. A non-causal draft block instead gives every row the same full extent. + +The kernel is replaced by a spy, so these tests assert on the ``block_table`` +and ``seq_lens`` the backend actually submits and need no GPU or Triton runtime. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from vllm.model_executor.layers.attention.mla_attention import QueryLenSupport +from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.attention.backends.mla.triton_mla import ( + TritonMLAImpl, + TritonMLAMetadataBuilder, +) + +NUM_HEADS = 16 +KV_LORA_RANK = 512 +QK_ROPE_HEAD_DIM = 64 +HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM +PAGE_SIZE = 16 + +# Committed context per request, including a fresh request at zero context +# where the whole KV extent is the block itself. A decode block's seq_len +# already counts the block, so seq_len = context + query_len. +CONTEXT_LENS = [0, 1, 37, 512] + + +def _seq_lens(query_len: int) -> list[int]: + return [c + query_len for c in CONTEXT_LENS] + + +def _make_impl(dcp_world_size: int = 1) -> TritonMLAImpl: + """Only the attributes ``forward_mqa`` reads; __init__ wants a full config.""" + impl = object.__new__(TritonMLAImpl) + impl.kv_lora_rank = KV_LORA_RANK + impl.scale = HEAD_SIZE**-0.5 + impl._sm_count = 304 + impl.dcp_world_size = dcp_world_size + return impl + + +def _make_metadata(seq_lens: list[int], query_len: int, causal: bool): + num_decodes = len(seq_lens) + return SimpleNamespace( + num_decodes=num_decodes, + num_decode_tokens=num_decodes * query_len, + max_seq_len=max(seq_lens), + causal=causal, + decode=SimpleNamespace( + # arange page ids so each request's window is an identifiable slice. + block_table=torch.arange(num_decodes * 64, dtype=torch.int32).reshape( + num_decodes, 64 + ), + seq_lens=torch.tensor(seq_lens, dtype=torch.int32), + ), + ) + + +def _run_forward_mqa( + seq_lens: list[int], + query_len: int, + causal: bool, + q_rows=None, + dcp_world_size: int = 1, +): + """Drive forward_mqa with the decode kernel spied out. + + Returns the ``(block_table, seq_lens)`` handed to ``decode_attention_fwd``. + """ + captured: dict = {} + + def spy(q, kv_cache, kv_c_cache, o, lse, block_table, b_seq_len, *args, **kwargs): + captured["block_table"] = block_table.detach().clone() + captured["seq_lens"] = b_seq_len.detach().clone() + + metadata = _make_metadata(seq_lens, query_len, causal) + rows = metadata.num_decode_tokens if q_rows is None else q_rows + q = torch.zeros(rows, NUM_HEADS, HEAD_SIZE, dtype=torch.bfloat16) + kv_cache = torch.zeros( + len(seq_lens) * 64, PAGE_SIZE, HEAD_SIZE, dtype=torch.bfloat16 + ) + layer = SimpleNamespace(_k_scale=torch.ones(1), layer_name="test") + + with patch("vllm.v1.attention.backends.mla.triton_mla.decode_attention_fwd", spy): + _make_impl(dcp_world_size).forward_mqa(q, kv_cache, metadata, layer) + + assert captured, "forward_mqa did not reach the decode kernel" + return captured + + +def test_multi_token_decode_flags_are_declared_together(): + """Capture probes max_query_len > 1, which only clears the assert in + build_for_cudagraph_capture once query_len_support raised the threshold.""" + assert ( + TritonMLAMetadataBuilder._cudagraph_support == AttentionCGSupport.UNIFORM_BATCH + ) + assert TritonMLAMetadataBuilder.query_len_support == QueryLenSupport.UNIFORM + + +@pytest.mark.parametrize("query_len", [2, 3, 4, 5, 8]) +def test_causal_block_rows_see_prefix_plus_own_position(query_len): + """Causal verify: row t sees seq_len - (query_len - 1) + t KV entries. + + Regression guard: the causal branch used to be skipped entirely, so a verify + token could attend the draft siblings it is supposed to be checking. Fails + on unmodified upstream, where no flatten happens at all for causal blocks. + """ + seq_lens = _seq_lens(query_len) + captured = _run_forward_mqa(seq_lens, query_len, causal=True) + + got = captured["seq_lens"].tolist() + want = [s - (query_len - 1) + t for s in seq_lens for t in range(query_len)] + assert got == want, ( + "causal flatten is not causal: verify row r*query_len+t must get " + f"seq_len_r - {query_len - 1} + t entries.\n" + f" seq_lens {seq_lens}\n got {got}\n expected {want}" + ) + + # Per-request invariants the arithmetic must hold regardless of query_len. + for r, seq_len in enumerate(seq_lens): + rows = got[r * query_len : (r + 1) * query_len] + assert rows == sorted(rows) and len(set(rows)) == query_len, ( + f"request {r} rows must strictly increase, got {rows}" + ) + assert rows[-1] == seq_len, "last row must see the full sequence" + assert rows[0] == seq_len - (query_len - 1) + assert all(n > 0 for n in rows), f"non-positive KV extent in {rows}" + + +def test_cudagraph_padding_rows_present_no_kv_extent(): + """Causal flattening pins cudagraph padding rows (seq_len 0) to exactly + zero KV extent, keeping them in the kernel's skipped branch.""" + query_len = 4 + seq_lens = _seq_lens(query_len) + [0] + captured = _run_forward_mqa(seq_lens, query_len, causal=True) + + padding = captured["seq_lens"].tolist()[-query_len:] + assert padding == [0] * query_len, ( + f"padding rows must present exactly zero KV extent, got {padding}" + ) + + +@pytest.mark.parametrize("query_len", [2, 4]) +def test_non_causal_block_rows_all_see_the_full_prefix(query_len): + """The DSpark draft block is generated in one pass, so no row is masked.""" + seq_lens = _seq_lens(query_len) + captured = _run_forward_mqa(seq_lens, query_len, causal=False) + + want = [s for s in seq_lens for _ in range(query_len)] + assert captured["seq_lens"].tolist() == want + + +@pytest.mark.parametrize("causal", [True, False]) +def test_block_table_is_expanded_to_match_q_rows(causal): + """The kernel indexes block_table by program_id, so it needs one row per + query token -- a short block_table is read out of bounds.""" + query_len = 4 + captured = _run_forward_mqa(_seq_lens(query_len), query_len, causal=causal) + + num_rows = len(CONTEXT_LENS) * query_len + assert captured["block_table"].shape[0] == num_rows + assert captured["seq_lens"].shape[0] == num_rows + # repeat_interleave, not repeat: a request's rows must stay adjacent. + for r in range(len(CONTEXT_LENS)): + block = captured["block_table"][r * query_len : (r + 1) * query_len] + assert torch.equal(block, block[:1].expand_as(block)) + + +@pytest.mark.parametrize("causal", [True, False]) +def test_single_token_decode_is_untouched(causal): + """query_len == 1 must short-circuit: ordinary decode sees no expansion.""" + seq_lens = _seq_lens(1) + captured = _run_forward_mqa(seq_lens, query_len=1, causal=causal) + + assert captured["seq_lens"].tolist() == seq_lens + assert captured["block_table"].shape[0] == len(seq_lens) + + +def test_expansion_factor_follows_the_q_row_count(): + """The kernel indexes block_table and seq_lens by program_id, so the + expansion must follow q rows, not the token/request counts.""" + num_rows = len(CONTEXT_LENS) * 4 + captured = _run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=num_rows) + + assert captured["block_table"].shape[0] == num_rows + assert captured["seq_lens"].shape[0] == num_rows + + +def test_non_uniform_block_is_rejected(): + """Q rows must divide evenly across the decode requests.""" + with pytest.raises(AssertionError, match="non-uniform decode block"): + _run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=6) + + +def test_causal_multi_token_decode_is_rejected_under_dcp(): + """Per-row extents offset the global seq_len, but DCP passes the rank-local + slice, so the causal arithmetic would silently address the wrong KV.""" + with pytest.raises(AssertionError, match="not supported with DCP"): + _run_forward_mqa(_seq_lens(4), query_len=4, causal=True, dcp_world_size=2) + + # Non-causal rows all take the same extent, which stays correct when local. + _run_forward_mqa(_seq_lens(4), query_len=4, causal=False, dcp_world_size=2) diff --git a/vllm/v1/attention/backends/mla/triton_mla.py b/vllm/v1/attention/backends/mla/triton_mla.py index 78787d5c26..3deb996a5b 100644 --- a/vllm/v1/attention/backends/mla/triton_mla.py +++ b/vllm/v1/attention/backends/mla/triton_mla.py @@ -6,7 +6,6 @@ from typing import ClassVar import torch import vllm.envs as envs -from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import ( @@ -14,6 +13,7 @@ from vllm.model_executor.layers.attention.mla_attention import ( MLACommonImpl, MLACommonMetadata, MLACommonMetadataBuilder, + QueryLenSupport, ) from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability @@ -26,7 +26,6 @@ from vllm.v1.attention.backend import ( MultipleOf, ) from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd -from vllm.v1.kv_cache_interface import KVCacheSpec from vllm.v1.worker.workspace import ( current_workspace_manager, is_workspace_manager_initialized, @@ -50,50 +49,16 @@ def _compute_num_kv_splits(max_seq_len: int, sm_count: int) -> int: class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): - _cudagraph_support: ClassVar[AttentionCGSupport] = ( - AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE - ) - # Non-causal DSpark block is flattened to one decode row per query token in - # forward_mqa, so no intra-block causal masking is required. + # forward_mqa flattens a uniform multi-token block to one decode row per + # query token, so causal and non-causal blocks both take the decode path. + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM supports_non_causal_multi_token_decode: ClassVar[bool] = True - @classmethod - def get_cudagraph_support( - cls, - vllm_config: VllmConfig, - kv_cache_spec: KVCacheSpec, - ) -> AttentionCGSupport: - """Report UNIFORM_BATCH where a non-causal multi-token block is served. - - ``_cudagraph_support`` is a class constant, so serving the DSpark - draft's (1 + num_spec) block through the decode path reports - UNIFORM_SINGLE_TOKEN_DECODE and, because the engine takes the minimum - over all attention groups, downgrades the *whole* engine off full - cudagraphs. ``forward_mqa`` flattens that block with - ``repeat_interleave`` on a Python int and performs no device->host - sync, so it does satisfy the UNIFORM_BATCH contract. - - ``non_causal_multi_token_decode`` is a KV-cache-group property, not a - per-layer one: ``MLAAttentionSpec.merge`` ORs it over every layer in - the group, so a group holding both a draft and its target reports it - for both. That is the same predicate ``__init__`` below already uses to - raise ``reorder_batch_threshold``, so the two stay consistent, but it - does mean this lifts a causal target sharing the draft's KV cache group - as well. - """ - if getattr(kv_cache_spec, "non_causal_multi_token_decode", False): - return AttentionCGSupport.UNIFORM_BATCH - return cls._cudagraph_support - def __init__(self, kv_cache_spec, layer_names, vllm_config, device): super().__init__(kv_cache_spec, layer_names, vllm_config, device) # DCP local sequence lengths are not advanced between draft steps. self.supports_draft_decode_metadata_update = self.dcp_world_size == 1 - # Only the non-causal DSpark draft group serves multi-token blocks via - # the decode path; raise its reorder threshold to the spec block length - # so full-cudagraph capture admits it. Causal usage stays single-token. - if getattr(self, "non_causal_multi_token_decode", False): - self._init_reorder_batch_threshold(1, supports_spec_as_decode=True) self._reserve_attn_logits_workspace() def update_draft_decode_metadata(self, _metadata: MLACommonMetadata) -> None: @@ -109,12 +74,12 @@ class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): """ if not is_workspace_manager_initialized(): return - # Decode reorder threshold is 1, so decode tokens <= max_num_seqs. - B = self.vllm_config.scheduler_config.max_num_seqs - # Non-causal DSpark draft flattens each request's block to query_len - # decode rows; cover max_num_seqs * block_len rows. - if getattr(self, "non_causal_multi_token_decode", False): - B *= self.reorder_batch_threshold + # forward_mqa flattens each request's block to query_len decode rows, + # and query_len is bounded by the reorder threshold. + B = ( + self.vllm_config.scheduler_config.max_num_seqs + * self.reorder_batch_threshold + ) # DCP all-gathers the query heads before forward_mqa. q_num_heads = self.num_heads * self.dcp_world_size max_splits = _compute_num_kv_splits( @@ -313,14 +278,32 @@ class TritonMLAImpl(MLACommonImpl[MLACommonMetadata]): block_table = attn_metadata.decode.block_table seq_lens = attn_metadata.decode.seq_lens - if not attn_metadata.causal: - # Non-causal DSpark block: flatten to one decode row per query token. - # Each row attends to the same committed KV prefix (per-row seq_lens) - # and never to sibling block tokens = non-causal block semantics. - # Mirrors FlashInferMLA's non-causal path. - query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes - if query_len > 1: - block_table = block_table.repeat_interleave(query_len, dim=0) + # decode_attention_fwd has no causal flag: it launches one program per + # row of q and reads that row's KV extent from seq_lens, so intra-block + # causality is expressed as per-row extents. Deriving query_len from the + # tensors the kernel indexes keeps the three row counts in step. + num_decodes = seq_lens.shape[0] + query_len, remainder = divmod(B, num_decodes) if num_decodes else (1, 0) + assert remainder == 0, ( + f"non-uniform decode block: {B} query rows over {num_decodes} requests" + ) + if query_len > 1: + block_table = block_table.repeat_interleave(query_len, dim=0) + if attn_metadata.causal: + # Per-row extents are offsets off the global sequence length; + # under DCP seq_lens is this rank's local slice instead. + assert self.dcp_world_size == 1, ( + "causal multi-token decode is not supported with DCP" + ) + # Row t attends the prefix plus block tokens 0..t. Clamp holds + # padding rows at extent 0; the kernel skips them either way + # (split_kv_end == split_kv_start). + offsets = torch.arange( + 1 - query_len, 1, device=seq_lens.device, dtype=seq_lens.dtype + ) + seq_lens = (seq_lens.unsqueeze(1) + offsets).flatten().clamp(min=0) + else: + # Non-causal draft block: every row sees the same prefix. seq_lens = seq_lens.repeat_interleave(query_len) # Run MQA — always pass layer scales. When KV cache is From 67a8a3f9269795d683868254d134a199c29aff4f Mon Sep 17 00:00:00 2001 From: shaohuaxi Date: Fri, 18 Sep 2026 02:05:13 -0700 Subject: [PATCH 085/143] [Bugfix][CPU] Fix macOS multimodal SHM cache initialization (#57142) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 子华 Co-authored-by: Codex --- tests/test_envs.py | 5 +++++ vllm/envs.py | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_envs.py b/tests/test_envs.py index 1ef40cde0c..dfd08a9ec2 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -18,6 +18,11 @@ from vllm.envs import ( from vllm.exceptions import VLLMValidationError +def test_object_storage_shm_default_name(): + """The generated name must fit macOS's shared-memory name limit.""" + assert len(envs._generate_shm_name()) <= 30 + + def test_getattr_without_cache(monkeypatch: pytest.MonkeyPatch): assert envs.VLLM_HOST_IP == "" assert envs.VLLM_PORT is None diff --git a/vllm/envs.py b/vllm/envs.py index 82d6e771b5..26c00c6da6 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -535,6 +535,14 @@ def get_vllm_port() -> int | None: raise ValueError(f"VLLM_PORT '{port}' must be a valid integer") from err +def _generate_shm_name() -> str: + import pybase64 + + # Fit macOS's 30-character limit without truncating the UUID. + encoded_uuid = pybase64.urlsafe_b64encode(uuid.uuid4().bytes).decode("ascii") + return "vllm_mm_" + encoded_uuid.rstrip("=") + + def get_env_or_set_default( env_name: str, default_factory: Callable[[], str], @@ -1943,7 +1951,7 @@ environment_variables: dict[str, Callable[[], Any]] = { # if not explicitly set. "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME": get_env_or_set_default( "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", - lambda: f"VLLM_OBJECT_STORAGE_SHM_BUFFER_{uuid.uuid4().hex}", + _generate_shm_name, ), # The size in MB of the buffers (NVL and RDMA) used by DeepEP "VLLM_DEEPEP_BUFFER_SIZE_MB": lambda: int( From 6df2b1a8c64a7956c51410cbfe1b786a69574263 Mon Sep 17 00:00:00 2001 From: meng-amd Date: Fri, 18 Sep 2026 17:52:46 +0800 Subject: [PATCH 086/143] [Quantization] Support native Quark W4A16 INT4/UINT4 exports in vLLM (#48606) Signed-off-by: HongWei Meng --- tests/quantization/test_quark.py | 440 +++++++++++++++++- .../model_executor/kernels/linear/__init__.py | 1 + .../layers/fused_moe/experts/triton_moe.py | 2 +- .../layers/quantization/auto_awq.py | 6 +- .../layers/quantization/quark/quark.py | 56 ++- .../layers/quantization/quark/quark_moe.py | 354 +++++++++++++- .../quantization/quark/schemes/__init__.py | 2 + .../quark/schemes/quark_w4a16_int4.py | 212 +++++++++ .../layers/quantization/quark/utils.py | 57 +++ 9 files changed, 1124 insertions(+), 6 deletions(-) create mode 100644 vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index ced034cb1a..39481456fd 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -58,12 +58,17 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501 from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501 QuarkMoEMethod, QuarkW4A8Fp8MoEMethod, + QuarkW4A16Int4MoEMethod, QuarkW8A8Fp8MoEMethod, QuarkW8A8Int8MoEMethod, ) -from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme +from vllm.model_executor.layers.quantization.quark.schemes import ( + QuarkScheme, + QuarkW4A16Int4, +) from vllm.model_executor.layers.quantization.quark.utils import ( QuarkQTensorHint, + canonicalize_quark_packed_int4, should_ignore_layer, ) from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( @@ -97,6 +102,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kNvfp4Static, ) from vllm.model_executor.models.llama import LlamaForCausalLM +from vllm.model_executor.models.utils import WeightsMapper from vllm.platforms import current_platform from vllm.transformers_utils.repo_utils import hf_api @@ -1876,3 +1882,435 @@ def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv(): ) assert target.loaded_shard is None # o_proj is a direct (non-fused) proj assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128)) + + +_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + + +def _quark_int4_config( + *, + pack_method: str = "reorder", + symmetric: bool = True, + exclude: list[str] | None = None, +) -> dict: + return { + "quant_method": "quark", + "export": {"pack_method": pack_method, "kv_cache_group": []}, + "global_quant_config": { + "weight": { + "dtype": "int4", + "group_size": 128, + "symmetric": symmetric, + } + }, + "exclude": exclude or [], + } + + +def _sign_extend_int4_nibbles(t: torch.Tensor) -> torch.Tensor: + mask = (t & 0x8).bool() + t = t.clone() + t[mask] = t[mask] | 0xF0 + return t + + +def _dequantize_quark_signed_awq_torch( + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + group_size: int, + *, + pack_reorder: bool = True, +) -> torch.Tensor: + bits = 4 + shifts = torch.arange(0, 32, bits, device=qweight.device) + iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + iweights = iweights.view(qweight.shape[0], -1) + zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + zeros = zeros.view(qzeros.shape[0], -1) + + if pack_reorder: + order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device) + else: + order = torch.arange(8, device=qweight.device) + iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape( + qweight.shape[0], -1 + ) + zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1) + iweights = _sign_extend_int4_nibbles(iweights & 0xF) + zeros = _sign_extend_int4_nibbles(zeros & 0xF) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def _dequantize_awq_unsigned_torch( + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + group_size: int, + *, + pack_reorder: bool = True, +) -> torch.Tensor: + bits = 4 + shifts = torch.arange(0, 32, bits, device=qweight.device) + iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + iweights = iweights.view(qweight.shape[0], -1) + zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + zeros = zeros.view(qzeros.shape[0], -1) + + if pack_reorder: + order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device) + else: + order = torch.arange(8, device=qweight.device) + iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape( + qweight.shape[0], -1 + ) + zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def _pack_int4_nibbles(nibbles: torch.Tensor, *, pack_reorder: bool) -> torch.Tensor: + pack_order = ( + torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=nibbles.device) + if pack_reorder + else torch.arange(8, device=nibbles.device) + ) + shifts = pack_order * 4 + return ((nibbles.to(torch.int64) & 0xF) << shifts).sum(dim=-1).to(torch.int32) + + +class TestQuarkInt4Format: + """Tests for Quark INT4 export format compatibility.""" + + def test_quark_order_pack_method_uses_native_int4_scheme(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(pack_method="order")) + weight_config = quant_config.quant_config["global_quant_config"] + weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config( + weight_config + ) + assert scheme_cls is QuarkW4A16Int4 + + scheme = quant_config.init_scheme( + scheme_cls, + weight_quant_key=weight_key, + activation_quant_key=act_key, + weight_config=weight_config["weight"], + ) + assert isinstance(scheme, QuarkW4A16Int4) + assert not scheme.pack_reorder + + def test_quark_int4_scheme_supports_asymmetric_weights(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + weight_config = quant_config.quant_config["global_quant_config"] + weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config( + weight_config + ) + assert scheme_cls is QuarkW4A16Int4 + + scheme = quant_config.init_scheme( + scheme_cls, + weight_quant_key=weight_key, + activation_quant_key=act_key, + weight_config=weight_config["weight"], + ) + assert isinstance(scheme, QuarkW4A16Int4) + assert not scheme.is_symmetric + + @pytest.mark.parametrize("missing_field", ["group_size", "symmetric"]) + def test_quark_int4_scheme_requires_weight_config_fields(self, missing_field): + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static, + ) + + weight_config = { + "dtype": "int4", + "group_size": 128, + "symmetric": True, + } + weight_config.pop(missing_field) + quant_config = QuarkConfig.from_config( + { + "quant_method": "quark", + "export": {"pack_method": "reorder", "kv_cache_group": []}, + "global_quant_config": {"weight": weight_config}, + "exclude": [], + } + ) + + with pytest.raises(ValueError, match=missing_field): + quant_config.init_scheme( + QuarkW4A16Int4, + weight_quant_key=kInt4Static, + activation_quant_key=None, + weight_config=weight_config, + ) + + def test_quark_int4_moe_uses_native_moe_method(self): + from unittest.mock import MagicMock, patch + + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4StaticAsym, + ) + + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + moe_config = type("MoeConfig", (), {})() + moe_config.has_bias = False + + mock_backend = MagicMock() + mock_experts_cls = MagicMock() + with patch( + "vllm.model_executor.layers.quantization.quark.quark_moe" + ".select_wna16_moe_backend", + return_value=(mock_backend, mock_experts_cls), + ): + method = QuarkW4A16Int4MoEMethod( + kInt4StaticAsym, + None, + quant_config.quant_config["global_quant_config"]["weight"], + quant_config.pack_method, + moe_config, + ) + + assert method.group_size == 128 + assert method.pack_reorder + assert method.use_wna16_backend + + def test_triton_wna16_experts_supports_asymmetric_int4(self): + from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( + TritonWNA16Experts, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, + ) + + for key in [kInt4Static, kInt4Static32, kInt4StaticAsym, kInt4Static32Asym]: + assert TritonWNA16Experts._supports_quant_scheme(key, None), ( + f"{key} should be supported by TritonWNA16Experts" + ) + + def test_quark_moe_loader_shards_w2_zero_point_across_tp_ranks(self): + """w2 zero-points are sharded along the intermediate dim per TP rank. + + Loading the same Quark checkpoint tensor at tensor_parallel_size=2 for + both ranks must jointly reconstruct the tensor_parallel_size=1 (full) + result, with no overlap or gap. This guards the custom loader's TP + split for the packed zero-point layout. + """ + + class _FakeMoEConfig: + has_bias = False + + def __init__(self, tp_size, tp_rank): + self.tp_size = tp_size + self.tp_rank = tp_rank + + class _FakeLayer: + def __init__(self, moe_config): + self.moe_config = moe_config + self.intermediate_size_per_partition = 64 + self.group_size_div_factor = 1 + + def load_w2_zero_point(raw_zp, tp_size, tp_rank): + from unittest.mock import MagicMock, patch + + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4StaticAsym, + ) + + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + moe_config = _FakeMoEConfig(tp_size, tp_rank) + with patch( + "vllm.model_executor.layers.quantization.quark.quark_moe" + ".select_wna16_moe_backend", + return_value=(MagicMock(), MagicMock()), + ): + method = QuarkW4A16Int4MoEMethod( + kInt4StaticAsym, + None, + quant_config.quant_config["global_quant_config"]["weight"], + quant_config.pack_method, + moe_config, + ) + + layer = _FakeLayer(moe_config) + # The loader converts the packed zero-point, then writes the + # per-rank slice into param.data[expert_id]. A single expert dim is + # enough; the last dim shrinks by tp_size after sharding. + param = torch.nn.Parameter( + torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8), + requires_grad=False, + ) + loader = method.get_weight_loader(layer, weight_loader=None) + loader( + param, + raw_zp.clone(), + weight_name="w2_weight_zero_point", + shard_id="w2", + expert_id=0, + ) + return param.data[0] + + # Quark-packed zero-point for one expert: rows = hidden // pack_factor, + # cols carry the (packed) intermediate dim, split evenly across ranks. + raw_zp = torch.randint(0, 256, (8, 16), dtype=torch.uint8) + + full = load_w2_zero_point(raw_zp, tp_size=1, tp_rank=0) + shard0 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=0) + shard1 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=1) + + expected = full.view(full.size(0), 2, -1) + torch.testing.assert_close(shard0, expected[:, 0]) + torch.testing.assert_close(shard1, expected[:, 1]) + + def test_quark_shared_expert_gate_keeps_quantized_tensors(self): + quant_config = QuarkConfig.from_config(_quark_int4_config()) + mapper = quant_config.get_cache_scale_mapper() + + output_names = { + name + for name, _ in mapper.apply( + [ + ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight", + torch.zeros(1), + ), + ( + "model.language_model.layers.0.mlp.shared_expert_gate" + ".weight_scale", + torch.zeros(1), + ), + ( + "model.language_model.layers.0.mlp.shared_expert_gate" + ".weight_zero_point", + torch.zeros(1), + ), + ] + ) + } + + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight" + in output_names + ) + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight_scale" + in output_names + ) + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight_zero_point" + in output_names + ) + + def test_quark_apply_mapper_updates_exclude_and_layer_quant_config(self): + quant_config = QuarkConfig.from_config( + { + **_quark_int4_config(), + "exclude": ["lm_head"], + "layer_quant_config": { + "model.language_model.layers.0.mlp.gate": { + "weight": {"dtype": "float16"}, + }, + }, + } + ) + quant_config.apply_vllm_mapper( + WeightsMapper( + orig_to_new_prefix={ + "lm_head": "language_model.lm_head", + "model.language_model.": "language_model.model.", + }, + ) + ) + + layer_quant_config = quant_config.quant_config["layer_quant_config"] + assert "language_model.model.layers.0.mlp.gate" in layer_quant_config + assert "model.language_model.layers.0.mlp.gate" not in layer_quant_config + assert quant_config.quant_config["exclude"] == [ + "language_model.lm_head", + ] + + def test_quark_apply_mapper_ignores_non_string_list_entries(self): + quant_config = QuarkConfig.from_config( + { + **_quark_int4_config(), + "algo_config": [ + {"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]} + ], + } + ) + quant_config.apply_vllm_mapper(WeightsMapper()) + + assert quant_config.quant_config["algo_config"] == [ + {"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]} + ] + + def test_quark_exclude_matches_via_mapper_prefix(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(exclude=["lm_head"])) + quant_config.apply_vllm_mapper( + WeightsMapper(orig_to_new_prefix={"lm_head": "language_model.lm_head"}) + ) + + exclude_layers = quant_config.quant_config["exclude"] + assert should_ignore_layer("language_model.lm_head", ignore=exclude_layers) + assert not should_ignore_layer( + "language_model.model.layers.0.mlp.gate", ignore=exclude_layers + ) + + +@pytest.mark.parametrize("symmetric", [False, True]) +@pytest.mark.parametrize("pack_method", ["order", "reorder"]) +def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric): + """Quark pack order is normalized to the layout expected by awq_* ops. + + This reuses the existing AWQ dequant/gemm kernels for compute only; loading + still goes through the native Quark quantization path, not AutoAWQ. + """ + pack_reorder = pack_method == "reorder" + group_size = 2 + packed_values = torch.tensor( + [ + [0, 1, 7, 8, 9, 15, 2, 14], + [15, 8, 0, 3, 12, 7, 1, 9], + ], + dtype=torch.int32, + ) + packed_zeros = torch.zeros((1, 8), dtype=torch.int32) + qweight = _pack_int4_nibbles(packed_values, pack_reorder=pack_reorder).view(2, 1) + qzeros = _pack_int4_nibbles(packed_zeros, pack_reorder=pack_reorder).view(1, 1) + scales = torch.ones((1, 8), dtype=torch.float16) + + dequantize = ( + _dequantize_quark_signed_awq_torch + if symmetric + else _dequantize_awq_unsigned_torch + ) + expected = dequantize( + qweight, + scales, + qzeros, + group_size, + pack_reorder=pack_reorder, + ) + canonical_weight = canonicalize_quark_packed_int4( + qweight, + pack_reorder=pack_reorder, + is_symmetric=symmetric, + ) + canonical_zero = canonicalize_quark_packed_int4( + qzeros, + pack_reorder=pack_reorder, + is_symmetric=symmetric, + ) + actual = _dequantize_awq_unsigned_torch( + canonical_weight, scales, canonical_zero, group_size + ) + + assert torch.equal(actual, expected) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 3c9c616d0c..8c57059e57 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -864,6 +864,7 @@ def choose_mp_linear_kernel( can_implement, failure_reason = kernel.can_implement(config) if can_implement: + logger.info_once("Using %s for mixed-precision linear", kernel.__name__) return kernel else: failure_reasons.append( diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 9f982644b5..da9666a66b 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -596,9 +596,9 @@ class TritonWNA16Experts(TritonExperts): ) -> bool: SUPPORTED_W = [ kInt4Static, + kInt4StaticAsym, kInt8Static, kInt4Static32, - kInt4StaticAsym, kInt4Static32Asym, # other group sizes? ] diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index 2c123b8768..71b858e919 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -93,7 +93,7 @@ def _replace_or_register_parameter( def _convert_awq_to_standard_format( layer: torch.nn.Module, w_q_name: str, - w_zp_name: str, + w_zp_name: str | None, size_bits: int, ) -> None: """Convert AWQ weight and zero-point tensors to standard GPTQ-like format. @@ -101,6 +101,7 @@ def _convert_awq_to_standard_format( AWQ packs qweight along the output dim with a non-standard bit order. This converts to standard bit order and repacks qweight along the input dim, matching the format expected by the MPLinearKernel framework. + If w_zp_name is None (symmetric quantization), only the weight is converted. """ pack_factor = 32 // size_bits mask = (1 << size_bits) - 1 @@ -139,6 +140,9 @@ def _convert_awq_to_standard_format( ) setattr(layer, w_q_name, new_param) + if w_zp_name is None: + return + # --- Convert qzeros: fix AWQ bit ordering and repack # AWQ qzeros: (G, N // pack) packed along dim 1, AWQ bit order # Target: (N // pack, G) packed along dim 0, standard bit order diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 88544fbe99..a3a3a36f00 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import ( QuarkOCP_MX, QuarkScheme, QuarkW4A8_MXFP4_FP8, + QuarkW4A16Int4, QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock, QuarkW8A8Int8, @@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import ( from vllm.model_executor.layers.quantization.quark.utils import ( QuarkQTensorHint, deep_compare, + parse_w4a16_int4_weight_config, should_ignore_layer, ) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( @@ -54,6 +56,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, kInt4W4A8StaticChannelSym, kInt8DynamicTensorAsym, kInt8DynamicTensorSym, @@ -172,8 +178,10 @@ class QuarkConfig(QuantizationConfig): quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {} for k, v in self.quant_config.items(): - if isinstance(v, list): + if isinstance(v, list) and all(isinstance(item, str) for item in v): quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_list(v) + elif isinstance(v, list): + quant_config_with_hf_to_vllm_mapper[k] = v elif isinstance(v, dict): quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_dict(v) else: @@ -215,6 +223,9 @@ class QuarkConfig(QuantizationConfig): weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, dynamic_mxfp4_quant=dynamic_mxfp4_quant, + weight_config=self._find_matched_config(prefix, type(layer)).get( + "weight" + ), ) layer.scheme = scheme return QuarkLinearMethod(self) @@ -227,6 +238,7 @@ class QuarkConfig(QuantizationConfig): method_cls=method_cls, weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, + layer_name=prefix, ) return None @@ -572,6 +584,26 @@ class QuarkConfig(QuantizationConfig): ) return QuantKeyMatch(True, activation_quant_key, weight_quant_key) + def _is_w4a16_int4( + self, + weight_quant: dict[str, Any] | None, + input_quant: dict[str, Any] | None, + ) -> QuantKeyMatch: + if weight_quant is None or input_quant is not None: + return QuantKeyMatch(False, None, None) + + is_int4 = weight_quant.get("dtype") in ("int4", "uint4") + is_packed = self.pack_method in ("order", "reorder") + if not (is_int4 and is_packed): + return QuantKeyMatch(False, None, None) + + group_size, is_symmetric = parse_w4a16_int4_weight_config(weight_quant) + if is_symmetric: + weight_key = kInt4Static32 if group_size == 32 else kInt4Static + else: + weight_key = kInt4Static32Asym if group_size == 32 else kInt4StaticAsym + return QuantKeyMatch(True, None, weight_key) + def _is_w4a8_mxfp4_fp8( self, weight_quant: dict[str, Any] | None, @@ -890,6 +922,12 @@ class QuarkConfig(QuantizationConfig): match.activation_quant_key, QuarkW8A8Int8, ) + elif match := self._is_w4a16_int4(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW4A16Int4, + ) elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config): return ( match.weight_quant_key, @@ -919,6 +957,7 @@ class QuarkConfig(QuantizationConfig): weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, dynamic_mxfp4_quant: bool = False, + weight_config: dict[str, Any] | None = None, ) -> "QuarkScheme": """Construct a Quark scheme selected by get_scheme_cls.""" if scheme_cls not in ( @@ -928,9 +967,24 @@ class QuarkConfig(QuantizationConfig): QuarkOCP_MX, QuarkNVFP4, QuarkW4A8_MXFP4_FP8, + QuarkW4A16Int4, ): raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}") + scheme: QuarkScheme + if scheme_cls is QuarkW4A16Int4: + normalized_weight_config = self._unwrap_single_quant_config(weight_config) + assert normalized_weight_config is not None + assert weight_quant_key is not None + scheme = QuarkW4A16Int4( + weight_quant_key=weight_quant_key, + activation_quant_key=activation_quant_key, + pack_method=self.pack_method, + weight_config=normalized_weight_config, + ) + self._check_scheme_supported(scheme.get_min_capability()) + return scheme + kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key} # These classes handle different possible `weight_quant_key`. diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index d2472524a5..3a9f6d6366 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import torch @@ -40,6 +40,12 @@ from vllm.model_executor.layers.fused_moe.oracle.int8 import ( make_int8_moe_quant_config, select_int8_moe_backend, ) +from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( + convert_to_wna16_moe_kernel_format, + make_wna16_moe_kernel, + make_wna16_moe_quant_config, + select_wna16_moe_backend, +) from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( TRITON_BACKENDS, Mxfp4MoeBackend, @@ -56,7 +62,10 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( make_nvfp4_moe_quant_config, select_nvfp4_moe_backend, ) -from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint +from vllm.model_executor.layers.quantization.quark.utils import ( + QuarkQTensorHint, + parse_w4a16_int4_weight_config, +) from vllm.model_executor.layers.quantization.utils.fp8_utils import ( validate_fp8_block_shape_moe, ) @@ -75,6 +84,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, kInt4W4A8StaticChannelSym, kInt8DynamicTensorAsym, kInt8DynamicTensorSym, @@ -102,6 +115,7 @@ logger = init_logger(__name__) __all__ = [ "QuarkMoEMethod", + "QuarkW4A16Int4MoEMethod", "QuarkW8A8Fp8MoEMethod", "QuarkOCP_MX_MoEMethod", "QuarkNvfp4MoEMethod", @@ -179,6 +193,12 @@ class QuarkMoEMethod(FusedMoEMethodBase): match.activation_quant_key, QuarkW8A8Fp8MoEMethod, ) + if match := quant_config._is_w4a16_int4(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW4A16Int4MoEMethod, + ) if match := quant_config._is_w_ocp_mx_a_x( weight_config, input_config, allow_static_fp8=True ): @@ -202,6 +222,7 @@ class QuarkMoEMethod(FusedMoEMethodBase): method_cls: type["QuarkMoEMethod"], weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, + layer_name: str | None = None, ) -> "QuarkMoEMethod": if weight_quant_key is None: raise RuntimeError("Unsupported FusedMoe scheme") @@ -226,10 +247,339 @@ class QuarkMoEMethod(FusedMoEMethodBase): return QuarkW8A8Int8MoEMethod( module.moe_config, weight_quant_key, activation_quant_key ) + elif method_cls is QuarkW4A16Int4MoEMethod: + assert layer_name is not None + layer_quant_config = quant_config._find_matched_config( + layer_name, type(module) + ) + weight_config = quant_config._unwrap_single_quant_config( + layer_quant_config.get("weight") + ) + if weight_config is None: + raise RuntimeError( + f"Missing weight quantization config for layer {layer_name}" + ) + return QuarkW4A16Int4MoEMethod( + weight_quant_key, + activation_quant_key, + weight_config, + quant_config.pack_method, + module.moe_config, + quant_config=quant_config, + ) else: raise RuntimeError("Unsupported FusedMoe scheme") +class QuarkW4A16Int4MoEMethod(QuarkMoEMethod): + """Quark packed INT4 weight-only MoE method.""" + + supported_activation_quant_keys: list[QuantKey | None] = [None] + supported_weight_quant_keys: list[QuantKey] = [ + kInt4Static, + kInt4Static32, + kInt4StaticAsym, + kInt4Static32Asym, + ] + + def __init__( + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, + weight_config: dict[str, Any], + pack_method: str, + moe: FusedMoEConfig, + quant_config: "QuarkConfig | None" = None, # type: ignore # noqa F821 + ): + super().__init__(moe, weight_quant_key, activation_quant_key) + + self.quant_config = quant_config + # group_size is read from the Quark config rather than weight_quant_key: + # the int4 QuantKeys only encode group_size 32 or -1, but Quark int4 + # checkpoints commonly use group_size 128, which the QuantKey cannot + # represent. Symmetry is unambiguous, so it comes from the key. + self.group_size = parse_w4a16_int4_weight_config(weight_config)[0] + self.is_symmetric = weight_quant_key.symmetric + self.bit8_pack_factor = 2 + self.pack_reorder = pack_method == "reorder" + + self.use_wna16_backend = True + self.wna16_backend, self.experts_cls = select_wna16_moe_backend( + config=self.moe, + weight_key=weight_quant_key, + quant_config=self.quant_config, + may_have_zp=not self.is_symmetric, + may_have_bias=False, + ) + + def create_weights( + self, + layer: RoutedExperts, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + group_size = self.group_size + group_size_div_factor = 1 + + while intermediate_size_per_partition % group_size or hidden_size % group_size: + group_size = group_size // 2 + group_size_div_factor *= 2 + assert group_size >= 32 + layer.group_size = group_size + layer.group_size_div_factor = group_size_div_factor + + strategy = FusedMoeWeightScaleSupported.GROUP.value + extra_weight_attrs.update({"quant_method": strategy, "is_transposed": False}) + + assert "weight_loader" in extra_weight_attrs + weight_loader = extra_weight_attrs["weight_loader"] + extra_weight_attrs["weight_loader"] = self.get_weight_loader( + layer, weight_loader + ) + + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // self.bit8_pack_factor, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition // self.bit8_pack_factor, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + w13_weight_scale = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + + w2_weight_scale = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size, + intermediate_size_per_partition // group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + w13_weight_zero_point = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition // self.bit8_pack_factor, + hidden_size // group_size, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_zero_point", w13_weight_zero_point) + set_weight_attrs(w13_weight_zero_point, extra_weight_attrs) + + w2_weight_zero_point = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size // self.bit8_pack_factor, + intermediate_size_per_partition // group_size, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_zero_point", w2_weight_zero_point) + set_weight_attrs(w2_weight_zero_point, extra_weight_attrs) + + def process_weights_after_loading(self, layer: RoutedExperts) -> None: + w13_qzeros = layer.w13_weight_zero_point if not self.is_symmetric else None + w2_qzeros = layer.w2_weight_zero_point if not self.is_symmetric else None + + converted = convert_to_wna16_moe_kernel_format( + backend=self.wna16_backend, + layer=layer, + quant_config=self.quant_config, + input_dtype=None, + w13=layer.w13_weight, + w2=layer.w2_weight, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w13_qzeros=w13_qzeros, + w2_qzeros=w2_qzeros, + ) + + if converted is None: + # Backend rewrote the layer's params in place (e.g. Humming). + self._setup_kernel(layer) + return + + ( + w13_qweight, + w2_qweight, + w13_scales, + w2_scales, + _, # w13_qzeros + _, # w2_qzeros + _, # w13_input_global_scale + _, # w2_input_global_scale + _, # w13_bias + _, # w2_bias + ) = converted + + replace_parameter(layer, "w13_weight", w13_qweight) + replace_parameter(layer, "w2_weight", w2_qweight) + replace_parameter(layer, "w13_weight_scale", w13_scales) + replace_parameter(layer, "w2_weight_scale", w2_scales) + + self._setup_kernel(layer) + + def _setup_kernel(self, layer: RoutedExperts) -> None: + assert self.experts_cls is not None + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + assert self.moe_quant_config is not None + self.moe_kernel = make_wna16_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + experts_cls=self.experts_cls, + backend=self.wna16_backend, + routing_tables=layer._expert_routing_tables(), + ) + + def get_fused_moe_quant_config( + self, layer: RoutedExperts + ) -> FusedMoEQuantConfig | None: + w1_zp = layer.w13_weight_zero_point if not self.is_symmetric else None + w2_zp = layer.w2_weight_zero_point if not self.is_symmetric else None + return make_wna16_moe_quant_config( + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w1_zp=w1_zp, + w2_zp=w2_zp, + group_size=layer.group_size, + num_bits=4, + ) + + def apply( + self, + layer: RoutedExperts, + x: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + shared_experts: SharedExperts | None, + shared_experts_input: torch.Tensor | None, + ) -> torch.Tensor: + assert self.moe_kernel is not None + return self.moe_kernel.apply( + hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=layer.activation, + global_num_experts=layer.global_num_experts, + apply_router_weight_on_input=layer.apply_router_weight_on_input, + expert_map=layer.expert_map, + shared_experts=shared_experts, + shared_experts_input=shared_experts_input, + ) + + def get_weight_loader(self, layer: RoutedExperts, weight_loader): + def convert_quark_tensor(tensor: torch.Tensor, tensor_type: str): + size0 = tensor.size(0) + tensor = tensor.view(torch.uint8) + + shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device) + tensor = (tensor[:, :, None] >> shifter) & 0xF + + if self.pack_reorder: + reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7] + tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order] + else: + tensor = tensor.view(-1, 8) + if self.is_symmetric: + tensor = tensor ^ 0x8 + tensor = tensor.view(size0, -1) + + tensor = tensor.T.contiguous() + if tensor_type == "weight": + tensor = tensor[:, 1::2] * 16 + tensor[:, ::2] + elif tensor_type == "zero_point": + tensor = tensor[1::2, :] * 16 + tensor[::2, :] + return tensor + + def quark_int4_weight_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: bool = False, + ): + loaded_weight = loaded_weight.to(param.data.device) + shard_size = layer.intermediate_size_per_partition + + if weight_name.endswith("_weight"): + loaded_weight = convert_quark_tensor(loaded_weight, "weight") + elif weight_name.endswith("_weight_zero_point"): + loaded_weight = convert_quark_tensor(loaded_weight, "zero_point") + elif weight_name.endswith("_weight_scale"): + loaded_weight = loaded_weight.T + + if layer.group_size_div_factor > 1 and ( + "weight_zero_point" in weight_name or "weight_scale" in weight_name + ): + loaded_weight = loaded_weight.repeat_interleave( + layer.group_size_div_factor, 1 + ) + + if "w13_weight_zero_point" in weight_name: + tensor = loaded_weight.view( + layer.moe_config.tp_size, -1, loaded_weight.size(1) + )[layer.moe_config.tp_rank] + if shard_id == "w1": + param.data[expert_id, : shard_size // 2] = tensor + else: + param.data[expert_id, shard_size // 2 :] = tensor + return True if return_success else None + elif "w2_weight_zero_point" in weight_name: + param.data[expert_id] = loaded_weight.view( + loaded_weight.size(0), layer.moe_config.tp_size, -1 + )[:, layer.moe_config.tp_rank] + return True if return_success else None + + return weight_loader( + param, + loaded_weight, + weight_name, + shard_id, + expert_id, + return_success=return_success, + ) + + return quark_int4_weight_loader + + class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): supported_activation_quant_keys = [ kFp8StaticTensorSym, diff --git a/vllm/model_executor/layers/quantization/quark/schemes/__init__.py b/vllm/model_executor/layers/quantization/quark/schemes/__init__.py index 4db1ba3845..a5857a776f 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/__init__.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/__init__.py @@ -5,6 +5,7 @@ from .quark_nvfp4 import QuarkNVFP4 from .quark_ocp_mx import QuarkOCP_MX from .quark_scheme import QuarkScheme from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8 +from .quark_w4a16_int4 import QuarkW4A16Int4 from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock from .quark_w8a8_int8 import QuarkW8A8Int8 @@ -14,6 +15,7 @@ __all__ = [ "QuarkW8A8Fp8PerBlock", "QuarkW8A8Int8", "QuarkOCP_MX", + "QuarkW4A16Int4", "QuarkW4A8_MXFP4_FP8", "QuarkNVFP4", ] diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py new file mode 100644 index 0000000000..cded0f80a9 --- /dev/null +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py @@ -0,0 +1,212 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math +from collections.abc import Callable +from typing import Any + +import torch + +from vllm.model_executor.kernels.linear import ( + MPLinearKernel, + MPLinearLayerConfig, + choose_mp_linear_kernel, +) +from vllm.model_executor.layers.quantization.auto_awq import ( + _convert_awq_to_standard_format, +) +from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import ( + QuarkScheme, +) +from vllm.model_executor.layers.quantization.quark.utils import ( + canonicalize_quark_packed_int4, + parse_w4a16_int4_weight_config, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, +) +from vllm.model_executor.parameter import ( + GroupQuantScaleParameter, + PackedvLLMParameter, +) +from vllm.scalar_type import scalar_types + + +class QuarkW4A16Int4(QuarkScheme): + """Quark packed INT4 weight-only linear scheme via MPLinearKernel.""" + + supported_activation_quant_keys: list[QuantKey | None] = [None] + supported_weight_quant_keys: list[QuantKey] = [ + kInt4Static, + kInt4Static32, + kInt4StaticAsym, + kInt4Static32Asym, + ] + + def __init__( + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, + pack_method: str, + weight_config: dict[str, Any], + ): + super().__init__(weight_quant_key, activation_quant_key) + + # group_size is read from the Quark config rather than weight_quant_key: + # the int4 QuantKeys only encode group_size 32 or -1, but Quark int4 + # checkpoints commonly use group_size 128, which the QuantKey cannot + # represent. Symmetry is unambiguous, so it comes from the key. + self.group_size = parse_w4a16_int4_weight_config(weight_config)[0] + self.pack_factor = 8 + self.pack_reorder = pack_method == "reorder" + self.is_symmetric = weight_quant_key.symmetric + self.quant_type = ( + scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4 + ) + self.kernel: MPLinearKernel | None = None + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + def create_weights( + self, + layer: torch.nn.Module, + output_partition_sizes: list[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs, + ): + input_size = kwargs["input_size"] + output_size = kwargs["output_size"] + group_size = ( + self.group_size if self.group_size != -1 else input_size_per_partition + ) + if input_size_per_partition % group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized weight shape. " + "This can be caused by too large tensor parallel size. " + f"input_size_per_partition={input_size_per_partition}, " + f"group_size={group_size}." + ) + + output_size_per_partition = sum(output_partition_sizes) + packed_output_size_per_partition = math.ceil( + output_size_per_partition / self.pack_factor + ) + layer.output_size_per_partition = output_size_per_partition + layer.packed_output_size_per_partition = packed_output_size_per_partition + + mp_linear_kernel_config = MPLinearLayerConfig( + full_weight_shape=(input_size, output_size), + partition_weight_shape=( + input_size_per_partition, + output_size_per_partition, + ), + weight_type=self.quant_type, + act_type=params_dtype, + group_size=group_size, + zero_points=not self.is_symmetric, + ) + kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config) + + def weight_scale_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + *args, + **loader_kwargs, + ) -> None: + if loaded_weight.shape[1] < param.data.shape[1]: + padded_weight = loaded_weight.new_zeros(param.data.shape) + padded_weight[:, : loaded_weight.shape[1]] = loaded_weight + loaded_weight = padded_weight + weight_loader(param, loaded_weight, *args, **loader_kwargs) + + weight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition, + packed_output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + ) + num_groups = input_size_per_partition // group_size + weight_zero_point = PackedvLLMParameter( + data=torch.zeros( + num_groups, + packed_output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + ) + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + num_groups, + packed_output_size_per_partition * self.pack_factor, + dtype=params_dtype, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_scale_loader, + ) + + layer.register_parameter("weight", weight) + layer.register_parameter("weight_zero_point", weight_zero_point) + layer.register_parameter("weight_scale", weight_scale) + + self.kernel = kernel_type( + mp_linear_kernel_config, + w_q_param_name="weight", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point" if not self.is_symmetric else None, + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.weight.data = canonicalize_quark_packed_int4( + layer.weight.data, + pack_reorder=self.pack_reorder, + is_symmetric=self.is_symmetric, + pack_factor=self.pack_factor, + ) + if not self.is_symmetric: + layer.weight_zero_point.data = canonicalize_quark_packed_int4( + layer.weight_zero_point.data, + pack_reorder=self.pack_reorder, + is_symmetric=self.is_symmetric, + pack_factor=self.pack_factor, + ) + output_size = layer.output_size_per_partition + packed_output_size = layer.packed_output_size_per_partition * self.pack_factor + if output_size < packed_output_size: + layer.weight_scale.data[:, output_size:].zero_() + + _convert_awq_to_standard_format( + layer, + "weight", + "weight_zero_point" if not self.is_symmetric else None, + self.quant_type.size_bits, + ) + assert self.kernel is not None + self.kernel.process_weights_after_loading(layer) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ): + assert self.kernel is not None + return self.kernel.apply_weights(layer, x, bias) diff --git a/vllm/model_executor/layers/quantization/quark/utils.py b/vllm/model_executor/layers/quantization/quark/utils.py index 48c978263e..ac455e38b8 100644 --- a/vllm/model_executor/layers/quantization/quark/utils.py +++ b/vllm/model_executor/layers/quantization/quark/utils.py @@ -66,6 +66,63 @@ def should_ignore_layer( return all(shards_ignored) +def parse_w4a16_int4_weight_config( + weight_config: Mapping[str, Any], +) -> tuple[int, bool]: + """Parse required W4A16 INT4/UINT4 weight fields from Quark config.""" + if "group_size" not in weight_config: + raise ValueError( + "Quark W4A16 INT4/UINT4 configs must specify weight.group_size" + ) + if "symmetric" not in weight_config: + raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric") + + group_size = weight_config["group_size"] + is_symmetric = weight_config["symmetric"] + if not isinstance(group_size, int) or group_size <= 0: + raise ValueError( + f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}" + ) + if not isinstance(is_symmetric, bool): + raise ValueError( + f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}" + ) + return group_size, is_symmetric + + +def canonicalize_quark_packed_int4( + packed_weight: torch.Tensor, + *, + pack_reorder: bool, + is_symmetric: bool, + pack_factor: int = 8, +) -> torch.Tensor: + """Convert Quark export nibble layout to AWQ checkpoint layout.""" + from vllm.model_executor.layers.quantization.auto_awq import ( + _REVERSE_AWQ_PACK_ORDER, + ) + + if pack_reorder: + source_order = torch.tensor( + _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32 + ) + else: + source_order = torch.arange( + pack_factor, device=packed_weight.device, dtype=torch.int32 + ) + target_order = torch.tensor( + _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32 + ) + source_shifts = source_order * 4 + target_shifts = target_order * 4 + + values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF + if is_symmetric: + values = values ^ 0x8 + packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1) + return packed.to(torch.int32) + + # utility for tensor dims > 2 cases def quark_quantize_weight_to_mxfp4(w: torch.Tensor): assert w.dtype == torch.bfloat16, ( From b34647906550f6e9dc65c55ecfb36b697d5e1f43 Mon Sep 17 00:00:00 2001 From: Yuwen Zhou Date: Fri, 18 Sep 2026 17:59:42 +0800 Subject: [PATCH 087/143] [CPU] Add CPU FP8 W8A8 linear/MoE support (#49942) Signed-off-by: yuwenzho Co-authored-by: Li, Jiang --- cmake/cpu_extension.cmake | 5 + csrc/cpu/cpu_attn.cpp | 13 +- csrc/cpu/cpu_isa.cpp | 20 + csrc/cpu/sgl-kernels/common.h | 83 ++ csrc/cpu/sgl-kernels/gemm.h | 54 +- csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp | 1093 +++++++++++++++++ csrc/cpu/sgl-kernels/moe.cpp | 100 +- csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp | 432 +++++++ csrc/cpu/torch_bindings.cpp | 37 +- tests/kernels/moe/test_cpu_quant_fused_moe.py | 198 +++ .../quantization/test_cpu_fp8_scaled_mm.py | 229 +++- vllm/_custom_ops.py | 44 + .../model_executor/kernels/linear/__init__.py | 4 + .../kernels/linear/scaled_mm/__init__.py | 2 + .../kernels/linear/scaled_mm/cpu.py | 310 ++++- .../layers/attention/mla_attention.py | 4 +- .../layers/fused_moe/experts/cpu_moe.py | 217 ++++ .../layers/fused_moe/oracle/fp8.py | 32 +- .../layers/quantization/online/fp8.py | 6 +- vllm/v1/attention/backends/mla/amx_mla.py | 87 +- 20 files changed, 2924 insertions(+), 46 deletions(-) create mode 100644 csrc/cpu/cpu_isa.cpp create mode 100644 csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp create mode 100644 csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index 6359a96887..cca2a98cee 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -488,6 +488,7 @@ set(VLLM_EXT_SRC "csrc/moe/dynamic_4bit_int_moe_cpu.cpp" "csrc/cpu/cpu_fused_moe.cpp" "csrc/cpu/cpu_attn.cpp" + "csrc/cpu/cpu_isa.cpp" "csrc/cpu/torch_bindings.cpp") if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64" AND VLLM_RVV_VLEN AND @@ -546,11 +547,13 @@ if (ENABLE_X86_ISA) "csrc/cpu/sgl-kernels/gemm.cpp" "csrc/cpu/sgl-kernels/gemm_int8.cpp" "csrc/cpu/sgl-kernels/gemm_fp8.cpp" + "csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp" "csrc/cpu/sgl-kernels/gemm_int4.cpp" "csrc/cpu/sgl-kernels/moe.cpp" "csrc/cpu/sgl-kernels/moe_int8.cpp" "csrc/cpu/sgl-kernels/moe_int4.cpp" "csrc/cpu/sgl-kernels/moe_fp8.cpp" + "csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp" "csrc/cpu/sgl-kernels/bmm.cpp" "csrc/cpu/sgl-kernels/decode.cpp" "csrc/cpu/sgl-kernels/extend.cpp" @@ -571,6 +574,7 @@ if (ENABLE_X86_ISA) "csrc/cpu/utils.cpp" "csrc/cpu/spec_decode_utils.cpp" "csrc/cpu/cpu_attn.cpp" + "csrc/cpu/cpu_isa.cpp" "csrc/cpu/dnnl_kernels.cpp" "csrc/cpu/mamba_cpu.cpp" "csrc/cpu/torch_bindings.cpp" @@ -587,6 +591,7 @@ if (ENABLE_X86_ISA) "csrc/cpu/utils.cpp" "csrc/cpu/spec_decode_utils.cpp" "csrc/cpu/cpu_attn.cpp" + "csrc/cpu/cpu_isa.cpp" "csrc/cpu/mamba_cpu.cpp" "csrc/cpu/dnnl_kernels.cpp" "csrc/cpu/torch_bindings.cpp" diff --git a/csrc/cpu/cpu_attn.cpp b/csrc/cpu/cpu_attn.cpp index b31a6faa01..ea0b4e49b2 100644 --- a/csrc/cpu/cpu_attn.cpp +++ b/csrc/cpu/cpu_attn.cpp @@ -1,16 +1,7 @@ #include "cpu_attn_dispatch_generated.h" -#if defined(__x86_64__) || defined(_M_X64) - #include -// Runtime check for AMX-FP8 (Diamond Rapids): CPUID leaf 7, subleaf 1, EAX[21]. -static bool runtime_has_amx_fp8() { - unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; - if (__get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx)) { - return (eax >> 21) & 1u; - } - return false; -} -#endif // x86_64 +// Runtime check for AMX-FP8, implemented in cpu_isa.cpp. +extern bool runtime_has_amx_fp8(); // Maps kv_cache_dtype string to Fp8KVCacheDataType enum. // "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2. diff --git a/csrc/cpu/cpu_isa.cpp b/csrc/cpu/cpu_isa.cpp new file mode 100644 index 0000000000..1c4a8faf19 --- /dev/null +++ b/csrc/cpu/cpu_isa.cpp @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +// AMX-FP8 is reported at CPUID leaf 0x1E, subleaf 1, EAX[4]. +#if defined(__x86_64__) || defined(_M_X64) + #include + +bool runtime_has_amx_fp8() { + unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; + if (__get_cpuid_max(0, nullptr) < 0x1e) return false; + if (__get_cpuid_count(0x1e, 1, &eax, &ebx, &ecx, &edx)) { + return (eax >> 4) & 1u; + } + return false; +} + +bool cpu_has_amx_fp8() { return runtime_has_amx_fp8(); } +#else +bool cpu_has_amx_fp8() { return false; } +#endif diff --git a/csrc/cpu/sgl-kernels/common.h b/csrc/cpu/sgl-kernels/common.h index 6f27ba2d5c..de1260bb33 100644 --- a/csrc/cpu/sgl-kernels/common.h +++ b/csrc/cpu/sgl-kernels/common.h @@ -12,11 +12,45 @@ #include #include #include +#include #if defined(_OPENMP) #include #endif +// Adapted from sglang: dispatch utilities for FP8 W8A8 kernels +#define AT_DISPATCH_OUT_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH( \ + TYPE, \ + NAME, \ + AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Float, out_t, __VA_ARGS__) \ + AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::BFloat16, out_t, __VA_ARGS__) \ + AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Half, out_t, __VA_ARGS__)) + +#define AT_DISPATCH_CASE_ENUM_NO_RETURN(VALUE, TYPE, HINT, ...) \ + case VALUE: { \ + constexpr TYPE HINT = VALUE; \ + __VA_ARGS__; \ + break; \ + } + +#define AT_DISPATCH_BOOL_NO_RETURN(VALUE, NAME, HINT, ...) \ + [&]() { \ + switch (VALUE) { \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(true, bool, HINT, __VA_ARGS__) \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(false, bool, HINT, __VA_ARGS__) \ + } \ + }() + +#define AT_DISPATCH_QUANT_MODE_NO_RETURN(MODE, NAME, HINT, ...) \ + [&]() { \ + switch (MODE) { \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_TENSOR, int, HINT, __VA_ARGS__) \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_ROW, int, HINT, __VA_ARGS__) \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_GROUP, int, HINT, __VA_ARGS__) \ + } \ + }() + namespace { // dispatch bool @@ -472,4 +506,53 @@ inline T* conditional_data_ptr(const std::optional& opt) { return opt.has_value() ? opt.value().data_ptr() : nullptr; } +// Adapted from sglang: M blocking utilities for FP8 W8A8 kernels +inline int64_t get_m_block(int64_t M) { + if (M <= 48) { + return M; + } else if (M < 64) { + return 32; + } else if (M < 96) { + return 64; + } else { + return 128; + } +} + +inline std::tuple get_m_blocking(int64_t M) { + bool parallel_on_M = M > 128; + int64_t block_m = get_m_block(M); + int64_t Mc = (M + block_m - 1) / block_m; + int64_t Mc_parallel = parallel_on_M ? Mc : 1; + return std::make_tuple(parallel_on_M, block_m, Mc, Mc_parallel); +} + +#if defined(CPU_CAPABILITY_AVX512) +template +void zero_buffer(T* data, int64_t size) { + const int32_t vec_size = at::vec::Vectorized::size(); + auto zero_vec = at::vec::Vectorized(0); + int64_t d = 0; + for (; d < size - (size % vec_size); d += vec_size) { + zero_vec.store(data + d); + } + if (d < size) { + zero_vec.store(data + d, size - d); + } +} +#else +template +void zero_buffer(T* data, int64_t size) { + memset(data, 0, sizeof(T) * size); +} +#endif + +#if defined(__x86_64__) +bool avx10_2_available() { + // __builtin_cpu_supports returns the masked feature bit rather than 0/1. + static const bool available = __builtin_cpu_supports("avx10.2"); + return available; +} +#endif + } // anonymous namespace diff --git a/csrc/cpu/sgl-kernels/gemm.h b/csrc/cpu/sgl-kernels/gemm.h index db7ad884d0..dc5f9c585d 100644 --- a/csrc/cpu/sgl-kernels/gemm.h +++ b/csrc/cpu/sgl-kernels/gemm.h @@ -94,7 +94,7 @@ enum class CPUActMethod : int { gelu_and_mul = 2, }; -enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4 }; +enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4, FP8_W8A8 = 5 }; constexpr bool operator==(CPUQuantMethod a, int64_t b) { return static_cast(a) == b; @@ -395,3 +395,55 @@ void tinygemm_kernel( bool brg, int64_t block_size_K, bool do_unpack = true); + +// FP8 W8A8 tinygemm kernel (defined in gemm_fp8_w8a8.cpp) +void tinygemm_kernel( + float* C, + const at::Float8_e4m3fn* A, + const float* scales_a, + const at::Float8_e4m3fn* B, + const float* scales_b, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc, + int64_t ldsa, + float* ukernel_buf, + at::BFloat16* dqA_buf, + at::BFloat16* dqB_buf); + +// FP8 W8A8 quantize activation (defined in gemm_fp8_w8a8.cpp) +std::tuple quantize_fp8e4m3_vec( + const at::Tensor& t, + bool channelwise, + c10::optional scale_opt); + +// FP8 W8A8 fused MoE kernel (defined in moe_fp8_w8a8.cpp) +template +void fused_experts_fp8_a8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + at::Float8_e4m3fn* __restrict__ A_tmp, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + float* __restrict__ Ukernel_tmp, + const at::Float8_e4m3fn* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ As, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad); diff --git a/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp b/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp new file mode 100644 index 0000000000..854566d682 --- /dev/null +++ b/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp @@ -0,0 +1,1093 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +#include +#include +#include + +#include "common.h" +#include "vec.h" + +// Runtime check for AMX-FP8, implemented in cpu_isa.cpp. +extern bool cpu_has_amx_fp8(); + +namespace { + +#define BLOCK_N 32 + +#define PER_TENSOR 1 +#define PER_ROW 2 +#define PER_GROUP 3 + +static bool cpublas_checked = false; +static bool cpublas_can_pack = false; + +bool cpublas_could_pack() { + // the could_pack check requires AMX support implicitly + if (cpublas_checked) { + return cpublas_can_pack; + } + cpublas_can_pack = cpu_has_amx_fp8() && at::native::cpublas::could_pack(at::kFloat8_e4m3fn); + cpublas_checked = true; + return cpublas_can_pack; +} + +#if defined(CPU_CAPABILITY_AVX512) +static void cvt_f8e4m3_to_bf16( + const at::Float8_e4m3fn* __restrict__ in, at::BFloat16* out, int64_t rows, int64_t cols, int64_t stride) { + if (stride == cols) { + // A contiguous buffer + size_t len = rows * cols; + size_t i = 0; + for (; i < len; i += 32) { + __m256i fp8_vec = _mm256_loadu_si256((__m256i*)&in[i]); + __m512bh bf16_vec = cvt_e4m3_bf16_intrinsic_no_nan(fp8_vec); + _mm512_storeu_si512((__m512i*)(out + i), (__m512i)bf16_vec); + } + for (; i < len; ++i) { + out[i] = (at::BFloat16)in[i]; + } + } else { + // Non-contiguous. Access each row with stride + TORCH_CHECK(stride > cols); + for (int r = 0; r < rows; ++r) { + size_t i = 0; + size_t vec_len = cols / 32 * 32; + for (; i < vec_len; i += 32) { + __m256i fp8_vec = _mm256_loadu_si256((__m256i*)&in[r * stride + i]); + __m512bh bf16_vec = cvt_e4m3_bf16_intrinsic_no_nan(fp8_vec); + _mm512_storeu_si512((__m512i*)(out + r * cols + i), (__m512i)bf16_vec); + } + for (; i < cols; ++i) { + out[r * cols + i] = (at::BFloat16)in[r * stride + i]; + } + } + } +} + +// accumulate and store result to buffer +// if act/wei are per_group quantized, apply scales +template +static void _accumulate_result( + float* __restrict__ output, + const float* __restrict__ input, + const float* __restrict__ scale_a, + const float* __restrict__ scale_b, + int M, + int ldi, + int ldo, + int ldsa = 1) { + float a_scale, b_scale; + __m512 va_scale; + __m512 vb_scale; + for (int m = 0; m < M; ++m) { + if constexpr (act_quant_mode == PER_GROUP) { + a_scale = *(scale_a + m * ldsa); + va_scale = _mm512_set1_ps(a_scale); + } + constexpr int N_UNROLL = N / 16; + c10::ForcedUnroll{}([&](auto i) { + constexpr int n = i * 16; + __m512 vc_f = _mm512_loadu_ps(input + m * ldi + n); + if constexpr (act_quant_mode == PER_GROUP) { + vc_f = _mm512_mul_ps(vc_f, va_scale); + } + if constexpr (wei_quant_mode == PER_GROUP) { + vb_scale = _mm512_loadu_ps(scale_b + n); + vc_f = _mm512_mul_ps(vc_f, vb_scale); + } + if constexpr (accum) { + __m512 vo = _mm512_loadu_ps(output + m * ldo + n); + _mm512_storeu_ps(output + m * ldo + n, _mm512_add_ps(vo, vc_f)); + } else { + _mm512_storeu_ps(output + m * ldo + n, vc_f); + } + }); + constexpr int tail_start = N / 16 * 16; + for (int n = tail_start; n < N; ++n) { + float dq_val = input[m * ldi + n]; + if constexpr (act_quant_mode == PER_GROUP) { + dq_val = dq_val * a_scale; + } + if constexpr (wei_quant_mode == PER_GROUP) { + b_scale = scale_b[n]; + dq_val = dq_val * b_scale; + } + if constexpr (accum) { + output[m * ldo + n] += dq_val; + } else { + output[m * ldo + n] = dq_val; + } + } + } +} + +// Store result to output buffer with dtype conversion +// If act/wei are per_row or per_tensor quantized, apply scales +// If bias is not null, add bias +template +inline void store_out( + const float* y_buf, + out_dtype* c_ptr, + int64_t M, + int64_t lda, + const float* scales_a, + const float* scales_b, + const float* bias) { + float a_scale = 1.0, b_scale = 1.0; + __m512 va_scale, vb_scale; + if constexpr (act_quant_mode == PER_TENSOR) { + a_scale = *scales_a; + } + if constexpr (wei_quant_mode == PER_TENSOR) { + b_scale = *scales_b; + vb_scale = _mm512_set1_ps(b_scale); + } + for (int i = 0; i < M; ++i) { + if constexpr (act_quant_mode == PER_ROW) { + a_scale = *(scales_a + i); + } + if constexpr (act_quant_mode != PER_GROUP) { + va_scale = _mm512_set1_ps(a_scale); + } + constexpr int N_UNROLL = N / 16; + c10::ForcedUnroll{}([&](auto idx) { + constexpr int j = idx * 16; + __m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j); + __m512 bias_vec = bias ? _mm512_loadu_ps(bias + j) : _mm512_setzero_ps(); + if constexpr (act_quant_mode != PER_GROUP) { + y_vec = _mm512_mul_ps(y_vec, va_scale); + } + if constexpr (wei_quant_mode == PER_ROW) { + vb_scale = _mm512_loadu_ps(scales_b + j); + } + if constexpr (wei_quant_mode != PER_GROUP) { + y_vec = _mm512_mul_ps(y_vec, vb_scale); + } + y_vec = _mm512_add_ps(y_vec, bias_vec); + if constexpr (std::is_same::value) { + _mm512_storeu_ps(c_ptr + i * lda + j, y_vec); + } else if constexpr (std::is_same::value) { + __m256i y_bf16_vec = at::vec::cvtfp32_bf16(y_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_bf16_vec); + } else if constexpr (std::is_same::value) { + __m256i y_fp16_vec = at::vec::cvtfp32_fp16(y_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_fp16_vec); + } else { + TORCH_CHECK(false, "Unsupported output dtype"); + } + }); + constexpr int tail_start = N / 16 * 16; + for (int j = tail_start; j < N; ++j) { + if constexpr (wei_quant_mode == PER_ROW) { + b_scale = scales_b[j]; + } + c_ptr[i * lda + j] = static_cast(y_buf[i * N + j] * a_scale * b_scale); + } + } // for M +} + +#else // no AVX512 + +static void cvt_f8e4m3_to_bf16( + const at::Float8_e4m3fn* __restrict__ in, at::BFloat16* out, int64_t rows, int64_t cols, int64_t stride) { + for (int r = 0; r < rows; ++r) { + for (int c = 0; c < cols; ++c) { + out[r * cols + c] = (at::BFloat16)in[r * stride + c]; + } + } +} + +// Store result to output buffer with dtype conversion +// If act/wei are per_row or per_tensor quantized, apply scales +// If bias is not null, add bias +template +inline void store_out( + const float* y_buf, + out_dtype* c_ptr, + int64_t M, + int64_t lda, + const float* scales_a, + const float* scales_b, + const float* bias) { + float a_scale = 1.0, b_scale = 1.0; + if constexpr (act_quant_mode == PER_TENSOR) { + a_scale = *scales_a; + } + if constexpr (wei_quant_mode == PER_TENSOR) { + b_scale = *scales_b; + } + for (int i = 0; i < M; ++i) { + if constexpr (act_quant_mode == PER_ROW) { + a_scale = *(scales_a + i); + } + for (int j = 0; j < N; ++j) { + if constexpr (wei_quant_mode == PER_ROW) { + b_scale = scales_b[j]; + } + c_ptr[i * lda + j] = static_cast(y_buf[i * N + j] * a_scale * b_scale); + } + } // for M +} + +#endif // CPU_CAPABILITY_AVX512 + +template +void _micro_gemm( + float* C, + const at::Float8_e4m3fn* A, + const float* scales_a, + const at::Float8_e4m3fn* B, + const float* scales_b, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc, + int64_t ldsa, + float* ukernel_buf, + at::BFloat16* dqA_buf, + at::BFloat16* dqB_buf) { + // If FP8 brgemm is not available, convert A/B to bf16 for computation + // Compute GEMM fp8 * fp8 -> fp32 (or bf16 * bf16 -> fp32) + // If per_group quant, apply scales. Otherwise, don't apply scales here + // Finally accumulate and store results +#if defined(CPU_CAPABILITY_AVX512) + if constexpr (cpublas_can_pack) { +#ifdef CPUBLAS_BRGEMM_F8F8F32 + at::native::cpublas::brgemm( + M, N, K, lda /*lda*/, N /*ldb*/, N /*ldc*/, false /* add_C */, A, B, ukernel_buf, true /* is_vnni */); +#else + cvt_f8e4m3_to_bf16(A, dqA_buf, M, K, lda); + cvt_f8e4m3_to_bf16(B, dqB_buf, K, N, N); + at::native::cpublas::brgemm( + M, N, K, K /*lda*/, N /*ldb*/, N /*ldc*/, false /* add_C */, dqA_buf, dqB_buf, ukernel_buf, true /* is_vnni */); +#endif + _mm_prefetch(B + N * (K + 128), _MM_HINT_T0); + _mm_prefetch(A + K + 128, _MM_HINT_T0); + _accumulate_result( + C, ukernel_buf, scales_a, scales_b, M, N /*ldi*/, ldc, ldsa); + } else +#endif + { + for (int64_t i = 0; i < M; ++i) { + for (int64_t j = 0; j < N; ++j) { + float sum = 0; + for (int64_t k = 0; k < K; ++k) { + sum += ((float)A[i * lda + k] * (float)B[k * N + j]); + } + if constexpr (act_quant_mode == PER_GROUP) { + sum *= scales_a[i * ldsa]; + } + if constexpr (wei_quant_mode == PER_GROUP) { + sum *= scales_b[j]; + } + C[i * ldc + j] += sum; + } + } + } +} + +template +void _float8_linear_impl( + const at::Tensor& input, + const at::Tensor& input_scales, + const at::Tensor& weight, + const at::Tensor& weight_scales, + const std::optional& bias, + at::Tensor& output) { + // input shape = [..., K] + // input is per token quantized + int64_t K = input.size(-1); + auto input_view = input.view({-1, K}); + int64_t M = input_view.size(0); + + // weight shape = [Nc, Kc, block_k, block_n] + // scales shape = [Nc, G, block_n] + int64_t Nc = weight.size(0); + int64_t Kc = wei_quant_mode != PER_GROUP ? 1 : weight.size(1); + int64_t block_k = wei_quant_mode != PER_GROUP ? weight.size(1) * weight.size(2) : weight.size(2); + constexpr int64_t block_n = BLOCK_N; + TORCH_CHECK(weight.size(3) == block_n, "Float8 linear: unexpected weight shape"); + int64_t N = Nc * block_n; + TORCH_CHECK(K == Kc * block_k, "Float8 linear: weight and input shapes mismatch"); + auto [parallel_on_M, block_m, Mc, Mc_parallel] = get_m_blocking(M); + + // scales shape = [Nc, G, block_n] + int64_t num_groups = wei_quant_mode == PER_TENSOR ? 1 : weight_scales.size(1); + TORCH_CHECK(K % num_groups == 0, "K should be divisible by num_groups"); + int64_t group_size = K / num_groups; + TORCH_CHECK(group_size % block_k == 0, "Float8 linear: group_size should be divisible by block_k"); + int64_t block_per_group = group_size / block_k; + TORCH_CHECK( + input_scales.numel() == 1 || input_scales.numel() == M || input_scales.numel() == M * num_groups, + "Float8 linear: unexpected input scales shape"); + auto ldsa = act_quant_mode == PER_TENSOR ? 0 : act_quant_mode == PER_ROW ? 1 : num_groups; + + const at::Float8_e4m3fn* a_ptr = input_view.data_ptr(); + const float* a_scales_ptr = input_scales.data_ptr(); + const at::Float8_e4m3fn* b_ptr = weight.data_ptr(); + const float* b_scales_ptr = weight_scales.data_ptr(); + out_dtype* c_ptr = output.data_ptr(); + const float* bias_ptr = bias.has_value() ? bias.value().data_ptr() : nullptr; + + int64_t block_size = block_m * block_n; + int64_t num_thread = at::get_num_threads(); + at::Tensor y_buffer = at::empty({num_thread, block_size}, output.options().dtype(at::kFloat)); + // Create buffer for brgemm output and dqA/dqB (optional) +#if defined(CPU_CAPABILITY_AVX512) + // buffer for brgemm output in float32 + int64_t buffer_size = block_size * 2; // float32 = bfloat16 * 2 +#ifndef CPUBLAS_BRGEMM_F8F8F32 + // buffers for dqA & dqB in bf16 + buffer_size += (block_k * block_n + block_m * block_k); +#endif + at::Tensor micro_gemm_buffer = at::empty({num_thread, buffer_size}, output.options().dtype(at::kBFloat16)); +#endif + + parallel_2d(Mc, Nc, [&](int64_t mc0, int64_t mc1, int64_t nc0, int64_t nc1) { + int tid = get_thread_num(); + float* y_buf = y_buffer.data_ptr() + tid * block_size; + at::BFloat16 *dqA_buffer = nullptr, *dqB_buffer = nullptr; + float* ukernel_buf = nullptr; +#if defined(CPU_CAPABILITY_AVX512) + at::BFloat16* micro_gemm_buf = micro_gemm_buffer.data_ptr() + tid * buffer_size; + ukernel_buf = reinterpret_cast(micro_gemm_buf); +#ifndef CPUBLAS_BRGEMM_F8F8F32 + dqA_buffer = micro_gemm_buf; + dqB_buffer = micro_gemm_buf + block_m * block_k; + ukernel_buf = reinterpret_cast(micro_gemm_buf + block_m * block_k + block_k * block_n); +#endif +#endif + + loop_2d(mc0, mc1, nc0, nc1, block_n * K, [&](int64_t mci, int64_t nc, int64_t) { + int64_t m_size = mci * block_m + block_m > M ? M - mci * block_m : block_m; + zero_buffer(y_buf, m_size * block_n); + for (int kci = 0; kci < Kc; ++kci) { + auto scales_a = a_scales_ptr + mci * block_m * num_groups + kci / block_per_group; + auto scales_b = b_scales_ptr + nc * block_n * num_groups + kci / block_per_group * block_n; + _micro_gemm( + /* C */ y_buf, + /* A */ a_ptr + mci * block_m * K + kci * block_k, + /* scales_a */ scales_a, + /* B */ b_ptr + (nc * Kc + kci) * block_n * block_k, + /* scales_b */ scales_b, + /* M */ m_size, + /* K */ block_k, + /* lda */ K, + /* ldc */ block_n, + /* ldsa */ ldsa, + /* ukernel_buf */ ukernel_buf, + /* dqA_buf */ dqA_buffer, + /* dqB_buf */ dqB_buffer); + } + auto scales_a = act_quant_mode == PER_TENSOR ? a_scales_ptr + : act_quant_mode == PER_ROW ? a_scales_ptr + mci * block_m + : nullptr; + auto scales_b = wei_quant_mode == PER_TENSOR ? b_scales_ptr + : wei_quant_mode == PER_ROW ? b_scales_ptr + nc * block_n + : nullptr; + auto bias_data = bias_ptr ? bias_ptr + nc * block_n : nullptr; + store_out( + y_buf, c_ptr + mci * block_m * N + nc * block_n, m_size, N /*lda*/, scales_a, scales_b, bias_data); + }); + + if constexpr (cpublas_can_pack) { + at::native::cpublas::brgemm_release(); + } + }); +} + +} // anonymous namespace + +void tinygemm_kernel( + float* C, + const at::Float8_e4m3fn* A, + const float* scales_a, + const at::Float8_e4m3fn* B, + const float* scales_b, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc, + int64_t ldsa, + float* ukernel_buf, + at::BFloat16* dqA_buf, + at::BFloat16* dqB_buf) { + // act_quant_mode = per row (2), wei_quant_mode = per group (3) + if (cpublas_could_pack()) { + _micro_gemm(C, A, scales_a, B, scales_b, M, K, lda, ldc, ldsa, ukernel_buf, dqA_buf, dqB_buf); + } else { + _micro_gemm(C, A, scales_a, B, scales_b, M, K, lda, ldc, ldsa, ukernel_buf, dqA_buf, dqB_buf); + } +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE() \ + void tinygemm_kernel( \ + float* C, \ + const at::Float8_e4m3fn* A, \ + const float* scales_a, \ + const at::Float8_e4m3fn* B, \ + const float* scales_b, \ + int64_t M, \ + int64_t K, \ + int64_t lda, \ + int64_t ldc, \ + int64_t ldsa, \ + float* ukernel_buf, \ + at::BFloat16* dqA_buf, \ + at::BFloat16* dqB_buf) + +INSTANTIATE_TINYGEMM_TEMPLATE(); +/* +return: packed_weight, packed_scales +*/ +std::tuple float8_linear_prepack_impl(const at::Tensor& weight, const at::Tensor& scales) { + // weight shape = [N, K] + // scales shape = [N, G] + TORCH_CHECK(weight.dim() == 2, "Float8 linear CPU: Weight should be a 2D tensor for packing"); + TORCH_CHECK(weight.size(1) % 2 == 0, "Float8 linear CPU: Weight should have even number of columns for packing"); + + auto new_scales = scales; + if (new_scales.dim() == 1) { + new_scales.unsqueeze_(1); + } + new_scales = new_scales.to(at::kFloat); + int N = weight.size(0); + int K = weight.size(1); + int G = scales.size(1); + int group_size = K / G; + int block_k = group_size > 128 ? 128 : group_size; + while (K % block_k != 0) { + block_k /= 2; + } + TORCH_CHECK( + block_k > 0 && block_k <= group_size, "Float8 linear CPU: Invalid block_k size, should be in (0, group_size]"); + constexpr int block_n = BLOCK_N; + int Nc = N / block_n; + int Kc = K / block_k; + + // Reorder weight to [N/block_n, K/block_k, block_k, block_n] + // Reorder scales to [N/block_n, G, block_n] + auto weight_view = weight.view({Nc, block_n, Kc, block_k}); + at::Tensor weight_reordered = weight_view.permute({0, 2, 3, 1}).contiguous(); + at::Tensor blocked_weight; + at::Tensor blocked_scales = new_scales.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous(); + +#if defined(CPU_CAPABILITY_AVX512) + if (cpublas_could_pack()) { +#ifdef CPUBLAS_BRGEMM_F8F8F32 + constexpr int vnni_size = 4; // for fp8 +#else + constexpr int vnni_size = 2; // for float16 +#endif + blocked_weight = at::empty({Nc, Kc, block_k, block_n}, weight.options()); + auto weight_ptr = reinterpret_cast(weight_reordered.data_ptr()); + auto blocked_weight_ptr = reinterpret_cast(blocked_weight.data_ptr()); + int64_t num_blocks = Nc * Kc; + at::parallel_for(0, num_blocks, 1, [&](int64_t begin, int64_t end) { + for (const auto i : c10::irange(begin, end)) { + auto in_ptr = weight_ptr + i * block_k * block_n; + auto out_ptr = blocked_weight_ptr + i * block_k * block_n; + + // Reorder weight block to VNNI + // plain shape = [block_k, block_n] + // packed shape = [block_k / VNNI_SIZE, block_n, VNNI_SIZE] viewed as [block_k, block_n] + constexpr int n_group_size = 8; + constexpr int n_group = block_n / n_group_size; // 4 + for (int nb = 0; nb < n_group; ++nb) { + for (int k = 0; k < block_k; k += vnni_size) { + for (int ni = 0; ni < n_group_size; ++ni) { + for (int ki = 0; ki < vnni_size; ++ki) { + int src_idx = nb * n_group_size + ni + (k + ki) * block_n; + int dst_idx = (nb * n_group_size + ni) * vnni_size + k * block_n + ki; + *(out_ptr + dst_idx) = *(in_ptr + src_idx); + } + } + } + } + } + }); + } else +#endif + { + blocked_weight = weight_reordered; + } + + return std::make_tuple(std::move(blocked_weight), std::move(blocked_scales)); +} + +// AVX512 optimized channel-wise max computation +inline void +compute_channel_max_avx512(const float* data, float* max_vals, int64_t num_channels, int64_t elements_per_channel) { + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + const float* channel_data = data + c * elements_per_channel; + float max_val = 0.0f; + + int64_t i = 0; + // Process 16 elements at a time + __m512 max_vec = _mm512_setzero_ps(); + + for (; i <= elements_per_channel - 16; i += 16) { + __m512 data_vec = _mm512_loadu_ps(&channel_data[i]); + __m512 abs_vec = _mm512_abs_ps(data_vec); + max_vec = _mm512_max_ps(max_vec, abs_vec); + } + + // Horizontal max of the vector + float hmax = _mm512_reduce_max_ps(max_vec); + max_val = std::max(max_val, hmax); + + // Handle remaining elements + for (; i < elements_per_channel; ++i) { + max_val = std::max(max_val, std::abs(channel_data[i])); + } + + max_vals[c] = max_val; + } + }); +} + +// AVX512 optimized scaling and clamping for channel-wise quantization +inline void scale_clamp_channelwise_avx512( + const float* src, + float* dst, + const float* scales, + int64_t num_channels, + int64_t elements_per_channel, + float quant_max, + float neg_quant_max) { + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + float scale_val = scales[c]; + float scale_reciprocal = 1.0f / scale_val; + + const float* channel_src = src + c * elements_per_channel; + float* channel_dst = dst + c * elements_per_channel; + + int64_t i = 0; + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(neg_quant_max); + + for (; i <= elements_per_channel - 16; i += 16) { + __m512 src_vec = _mm512_loadu_ps(&channel_src[i]); + __m512 scaled_vec = _mm512_mul_ps(src_vec, scale_recip_vec); + __m512 clamped_vec = _mm512_min_ps(_mm512_max_ps(scaled_vec, neg_quant_max_vec), quant_max_vec); + _mm512_storeu_ps(&channel_dst[i], clamped_vec); + } + + // Handle remaining elements + for (; i < elements_per_channel; ++i) { + float scaled = channel_src[i] * scale_reciprocal; + channel_dst[i] = std::clamp(scaled, neg_quant_max, quant_max); + } + } + }); +} + +// AVX512 optimized scaling and clamping for global quantization +inline void scale_clamp_global_avx512( + const float* src, float* dst, int64_t size, float scale_reciprocal, float quant_max, float neg_quant_max) { + if (size <= 0) return; + + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(neg_quant_max); + + at::parallel_for(0, size, 4096, [&](int64_t start, int64_t end) { + int64_t block_size = end - start; + const float* block_src = src + start; + float* block_dst = dst + start; + + int64_t i = 0; + for (; i <= block_size - 16; i += 16) { + __m512 src_vec = _mm512_loadu_ps(&block_src[i]); + __m512 scaled_vec = _mm512_mul_ps(src_vec, scale_recip_vec); + __m512 clamped_vec = _mm512_min_ps(_mm512_max_ps(scaled_vec, neg_quant_max_vec), quant_max_vec); + _mm512_storeu_ps(&block_dst[i], clamped_vec); + } + + // Handle remaining elements in block + for (; i < block_size; ++i) { + float scaled = block_src[i] * scale_reciprocal; + block_dst[i] = std::clamp(scaled, neg_quant_max, quant_max); + } + }); +} + +std::tuple +_quantize_fp8e4m3(const at::Tensor& t, bool channelwise, c10::optional scale_opt = c10::nullopt) { + constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max + constexpr float eps = std::numeric_limits::epsilon(); + + // Ensure input is contiguous and in float32 + auto t_float = t.to(at::ScalarType::Float).contiguous(); + at::Tensor qt; + at::Tensor scale_tensor; + + if (channelwise) { + // Channel-wise quantization with AVX512 optimization + int64_t num_channels = t_float.size(0); + int64_t elements_per_channel = t_float.numel() / num_channels; + + if (elements_per_channel * num_channels != t_float.numel()) { + throw std::runtime_error("Tensor must be divisible by number of channels for channel-wise quantization"); + } + + // Allocate scale tensor + scale_tensor = at::empty({num_channels}, t_float.options()); + float* scale_data = scale_tensor.data_ptr(); + + // Compute channel-wise max using AVX512 + compute_channel_max_avx512(t_float.data_ptr(), scale_data, num_channels, elements_per_channel); + + // Apply quant_max and EPS + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + scale_data[c] = std::max(scale_data[c] / quant_max, eps); + } + }); + + // Create output tensor for quantized values + qt = at::empty_like(t_float); + + // Scale and clamp using AVX512 + scale_clamp_channelwise_avx512( + t_float.data_ptr(), + qt.data_ptr(), + scale_data, + num_channels, + elements_per_channel, + quant_max, + -quant_max); + } else { + // Global quantization with AVX512 optimization + if (!scale_opt.has_value()) { + throw std::runtime_error("Scale must be provided for non-channelwise quantization in AVX512 version"); + } + + scale_tensor = scale_opt.value().to(at::ScalarType::Float).contiguous(); + + // Handle scalar scale case + float scale_val; + if (scale_tensor.numel() == 1) { + scale_val = std::max(scale_tensor.item(), eps); + } else { + // Compute max of scale tensor if it's not scalar + float* scale_data = scale_tensor.data_ptr(); + __m512 max_vec = _mm512_set1_ps(eps); + + int64_t i = 0; + for (; i <= scale_tensor.numel() - 16; i += 16) { + __m512 scale_vec = _mm512_loadu_ps(&scale_data[i]); + max_vec = _mm512_max_ps(max_vec, scale_vec); + } + + float hmax = _mm512_reduce_max_ps(max_vec); + for (; i < scale_tensor.numel(); ++i) { + hmax = std::max(hmax, scale_data[i]); + } + scale_val = std::max(hmax, eps); + } + + scale_tensor = at::tensor({scale_val}, scale_tensor.options()); + float scale_reciprocal = 1.0f / scale_val; + + // Create output tensor + qt = at::empty_like(t_float); + + // Scale and clamp using AVX512 + scale_clamp_global_avx512( + t_float.data_ptr(), qt.data_ptr(), t_float.numel(), scale_reciprocal, quant_max, -quant_max); + } + + // Final conversion to FP8 E4M3FN using ATen's native conversion + qt = at::_to_copy(qt, at::ScalarType::Float8_e4m3fn); + + return std::make_tuple(qt, scale_tensor); +} + +inline __m128i cvtfp32_fp8e4m3(__m512& src) { + // cvt 16x32 from fp32 to fp8 e4m3 + const __m512i sign_mask = _mm512_set1_epi32(0x80000000); + const __m512i fp8_max = _mm512_set1_epi32(UINT32_C(1087) << 20); + const __m512i denorm_thresh = _mm512_set1_epi32(UINT32_C(121) << 23); + const __m512i denorm_mask = _mm512_set1_epi32(UINT32_C(141) << 23); + const __m512i bias_part1 = _mm512_set1_epi32((uint32_t)(7 - 127) << 23); + const __m512i rounding_bias = _mm512_set1_epi32(0x7FFFF); + __m512i f_bits = _mm512_castps_si512(src); + // Extract and save sign + __m512i sign = _mm512_and_epi32(f_bits, sign_mask); + f_bits = _mm512_xor_epi32(f_bits, sign); + + // Prepare result containers + __m512i result = _mm512_setzero_si512(); + + // Step 1: Handle case of overflow + // (f_bits >= fp8_max): set result = 0x7f + __mmask16 overflow_mask = _mm512_cmpge_epu32_mask(f_bits, fp8_max); + if (overflow_mask) { + result = _mm512_mask_set1_epi32(result, overflow_mask, 0x7f); + } + + // Step 2: Handle small numbers (denormals) + // Small numbers (f_bits < denorm_thresh) + __mmask16 denorm_thresh_mask = _mm512_cmplt_epu32_mask(f_bits, denorm_thresh); + + if (denorm_thresh_mask) { + __m512 small_input = _mm512_castsi512_ps(f_bits); + __m512 small_denorm = _mm512_add_ps(small_input, _mm512_castsi512_ps(denorm_mask)); + __m512i small_denorm_bits = _mm512_castps_si512(small_denorm); + __m512i small_result = _mm512_sub_epi32(small_denorm_bits, denorm_mask); + result = _mm512_mask_mov_epi32(result, denorm_thresh_mask, small_result); + } + + // Step 3: Handle normal numbers + __mmask16 normal_mask = ~(overflow_mask | denorm_thresh_mask); + + if (normal_mask) { + // mant_odd = (f_bits >> 20) & 1 + __m512i mant_odd = _mm512_and_epi32(_mm512_srli_epi32(f_bits, 20), _mm512_set1_epi32(1)); + // f_bits += bias_part1 + rounding_bias + __m512i rounded = _mm512_add_epi32(f_bits, bias_part1); + rounded = _mm512_add_epi32(rounded, rounding_bias); + // Add mant_odd + rounded = _mm512_add_epi32(rounded, mant_odd); + // Shift right by 20 bits + __m512i normal_result = _mm512_srli_epi32(rounded, 20); + result = _mm512_mask_mov_epi32(result, normal_mask, normal_result); + } + + // Merge back the sign + __m512i sign_shifted = _mm512_srli_epi32(sign, 24); + result = _mm512_or_epi32(result, sign_shifted); + + // Now result is 16 x 32-bit integers, but we only need 8-bit for each + __m512i packed = _mm512_and_si512(result, _mm512_set1_epi32(0xFF)); + + // Narrow 32-bit integers to 8-bit + return _mm512_cvtepi32_epi8(packed); +} + +__attribute__((target("avx10.2"))) inline __m128i cvtfp32_fp8e4m3_avx10_2(__m512& src) { + __m256i f16_vec = _mm512_cvt_roundps_ph(src, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + return _mm256_cvtph_hf8(_mm256_castsi256_ph(f16_vec)); +} + +// Carries the whole loop rather than just the conversion, as GCC cannot inline an avx10.2 +// function into a caller lacking that target, so branching per vector is costly. +__attribute__((target("avx10.2"))) void quantize_bf16_block_avx10_2( + const at::BFloat16* __restrict__ src, at::Float8_e4m3fn* __restrict__ dst, int64_t count, float scale_reciprocal) { + constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(-quant_max); + + for (int64_t i = 0; i <= count - 32; i += 32) { + __m256i src_vec1 = _mm256_loadu_si256(reinterpret_cast(&src[i])); + __m256i src_vec2 = _mm256_loadu_si256(reinterpret_cast(&src[i + 16])); + + __m512 fp32_vec1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec1), 16)); + __m512 fp32_vec2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec2), 16)); + + __m512 scaled_vec1 = _mm512_mul_ps(fp32_vec1, scale_recip_vec); + __m512 clamped_vec1 = _mm512_min_ps(_mm512_max_ps(scaled_vec1, neg_quant_max_vec), quant_max_vec); + + __m512 scaled_vec2 = _mm512_mul_ps(fp32_vec2, scale_recip_vec); + __m512 clamped_vec2 = _mm512_min_ps(_mm512_max_ps(scaled_vec2, neg_quant_max_vec), quant_max_vec); + + __m128i fp8_vec1 = cvtfp32_fp8e4m3_avx10_2(clamped_vec1); + __m128i fp8_vec2 = cvtfp32_fp8e4m3_avx10_2(clamped_vec2); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), fp8_vec1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16), fp8_vec2); + } +} + +std::tuple _quantize_fp8e4m3_bf16_per_tensor_no_scale(const at::Tensor& t) { + constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max + constexpr float eps = std::numeric_limits::epsilon(); + + // Input validation and preparation + assert(t.scalar_type() == at::ScalarType::BFloat16); + auto t_bf16 = t.contiguous(); + int64_t num_channels = t_bf16.size(0); + int64_t elements_per_channel = t_bf16.numel() / num_channels; + assert(elements_per_channel % 32 == 0); // do not consider tile currently + at::Tensor quant_t = at::empty(t_bf16.sizes(), t_bf16.options().dtype(at::kFloat8_e4m3fn)); + + // Allocate output tensors + at::Tensor scale_tensor = at::empty({num_channels}, t_bf16.options().dtype(at::ScalarType::Float)); + float* scale_data = scale_tensor.data_ptr(); + + // Unified processing: compute max, apply scale, and quantize in single pass + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + const at::BFloat16* channel_src = t_bf16.data_ptr() + c * elements_per_channel; + at::Float8_e4m3fn* quant_dst = quant_t.data_ptr() + c * elements_per_channel; + + // Step 1: Compute channel-wise max using AVX512 + float channel_max = 0.0f; + int64_t i = 0; + + // Process 32 elements at a time for max computation + for (; i <= elements_per_channel - 32; i += 32) { + // Load 32 BF16 values (2x 256-bit vectors) + __m256i src_vec1 = _mm256_loadu_si256(reinterpret_cast(&channel_src[i])); + __m256i src_vec2 = _mm256_loadu_si256(reinterpret_cast(&channel_src[i + 16])); + + // Convert BF16 to FP32 + __m512i fp32_int1 = _mm512_cvtepu16_epi32(src_vec1); + __m512 fp32_vec1 = _mm512_castsi512_ps(_mm512_slli_epi32(fp32_int1, 16)); + + __m512i fp32_int2 = _mm512_cvtepu16_epi32(src_vec2); + __m512 fp32_vec2 = _mm512_castsi512_ps(_mm512_slli_epi32(fp32_int2, 16)); + + // Compute absolute values + __m512 abs_vec1 = _mm512_abs_ps(fp32_vec1); + __m512 abs_vec2 = _mm512_abs_ps(fp32_vec2); + + // Find max in each vector + float max1 = _mm512_reduce_max_ps(abs_vec1); + float max2 = _mm512_reduce_max_ps(abs_vec2); + channel_max = std::max(channel_max, std::max(max1, max2)); + } + + // Step 2: Apply quant_max and EPS to compute scale + float scale_val = std::max(channel_max / quant_max, eps); + float scale_reciprocal = 1.0f / scale_val; + scale_data[c] = scale_val; + + // Step 3: Scale and clamp using AVX512 (reuse the same loop structure) + if (avx10_2_available()) { + quantize_bf16_block_avx10_2(channel_src, quant_dst, elements_per_channel, scale_reciprocal); + continue; + } + i = 0; + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(-quant_max); + + for (; i <= elements_per_channel - 32; i += 32) { + // Load 32 BF16 values + __m256i src_vec1 = _mm256_loadu_si256(reinterpret_cast(&channel_src[i])); + __m256i src_vec2 = _mm256_loadu_si256(reinterpret_cast(&channel_src[i + 16])); + + // Convert BF16 to FP32 + __m512 fp32_vec1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec1), 16)); + __m512 fp32_vec2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec2), 16)); + + // Scale and clamp + __m512 scaled_vec1 = _mm512_mul_ps(fp32_vec1, scale_recip_vec); + __m512 clamped_vec1 = _mm512_min_ps(_mm512_max_ps(scaled_vec1, neg_quant_max_vec), quant_max_vec); + + __m512 scaled_vec2 = _mm512_mul_ps(fp32_vec2, scale_recip_vec); + __m512 clamped_vec2 = _mm512_min_ps(_mm512_max_ps(scaled_vec2, neg_quant_max_vec), quant_max_vec); + + __m128i fp8_vec1 = cvtfp32_fp8e4m3(clamped_vec1); + __m128i fp8_vec2 = cvtfp32_fp8e4m3(clamped_vec2); + _mm_storeu_si128(reinterpret_cast<__m128i*>(quant_dst + i), fp8_vec1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(quant_dst + i + 16), fp8_vec2); + } + } + }); + + return std::make_tuple(quant_t, scale_tensor); +} + +std::tuple +_quantize_fp8e4m3_bf16_per_tensor_with_scale(const at::Tensor& t, at::Tensor& scale_tensor) { + constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max + constexpr float eps = std::numeric_limits::epsilon(); + + // Input validation + assert(t.scalar_type() == at::ScalarType::BFloat16); + + auto t_bf16 = t.contiguous(); + auto scale_tensor_contig = scale_tensor.contiguous().to(at::ScalarType::Float); + + // Get scale value (handle scalar or tensor) + float scale_val; + if (scale_tensor_contig.numel() == 1) { + scale_val = scale_tensor_contig.item(); + } else { + // Take max if scale is a tensor (though should be scalar for global quantization) + scale_val = scale_tensor_contig.max().item(); + } + + // Apply EPS to scale + scale_val = std::max(scale_val, eps); + float scale_reciprocal = 1.0f / scale_val; + + at::Tensor scale_output = at::tensor({scale_val}, scale_tensor_contig.options()); + + // Apply scale and clamp using AVX512 + const at::BFloat16* src_data = t_bf16.data_ptr(); + at::Tensor quant_t = at::empty(t_bf16.sizes(), t_bf16.options().dtype(at::kFloat8_e4m3fn)); + int64_t total_elements = t_bf16.numel(); + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(-quant_max); + int64_t num_channels = t_bf16.size(0); + int64_t elements_per_channel = total_elements / num_channels; + assert(elements_per_channel % 32 == 0); // do not consider tile currently + // Process in parallel blocks + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + const at::BFloat16* src_data = t_bf16.data_ptr() + c * elements_per_channel; + at::Float8_e4m3fn* quant_t_data = quant_t.data_ptr() + c * elements_per_channel; + + if (avx10_2_available()) { + quantize_bf16_block_avx10_2(src_data, quant_t_data, elements_per_channel, scale_reciprocal); + continue; + } + + int64_t i = 0; + // Process 32 elements at a time using AVX512 + for (; i <= elements_per_channel - 32; i += 32) { + // Load 32 BF16 values (2x 256-bit vectors) + __m256i src_vec1 = _mm256_loadu_si256(reinterpret_cast(&src_data[i])); + __m256i src_vec2 = _mm256_loadu_si256(reinterpret_cast(&src_data[i + 16])); + + // Convert BF16 to FP32 + __m512 fp32_vec1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec1), 16)); + __m512 fp32_vec2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec2), 16)); + + // Scale and clamp + __m512 scaled_vec1 = _mm512_mul_ps(fp32_vec1, scale_recip_vec); + __m512 clamped_vec1 = _mm512_min_ps(_mm512_max_ps(scaled_vec1, neg_quant_max_vec), quant_max_vec); + + __m512 scaled_vec2 = _mm512_mul_ps(fp32_vec2, scale_recip_vec); + __m512 clamped_vec2 = _mm512_min_ps(_mm512_max_ps(scaled_vec2, neg_quant_max_vec), quant_max_vec); + + __m128i fp8_vec1 = cvtfp32_fp8e4m3(clamped_vec1); + __m128i fp8_vec2 = cvtfp32_fp8e4m3(clamped_vec2); + _mm_storeu_si128(reinterpret_cast<__m128i*>(quant_t_data + i), fp8_vec1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(quant_t_data + i + 16), fp8_vec2); + } + } + }); + + return std::make_tuple(quant_t, scale_output); +} + +std::tuple +_quantize_fp8e4m3_vec(const at::Tensor& t, bool channelwise, c10::optional scale_opt) { + if (channelwise) { + return _quantize_fp8e4m3_bf16_per_tensor_no_scale(t); + } else { + assert(scale_opt.has_value()); + return _quantize_fp8e4m3_bf16_per_tensor_with_scale(t, scale_opt.value()); + } +} + +// Public wrapper for torch op registration +std::tuple +quantize_fp8e4m3_vec(const at::Tensor& t, bool channelwise, c10::optional scale_opt) { + return _quantize_fp8e4m3_vec(t, channelwise, scale_opt); +} + +at::Tensor fp8_scaled_mm_with_quant( + const at::Tensor& act, + const std::optional& act_scales, + bool channelwise, + const at::Tensor& weight, + const at::Tensor& weight_scales, + const std::optional& bias, + at::ScalarType output_dtype) { + // TODO: refine here to use tensor ptr and in below dispatch api. + std::tuple quant_act = act.scalar_type() == at::ScalarType::BFloat16 + ? _quantize_fp8e4m3_vec(act, channelwise, act_scales) + : _quantize_fp8e4m3(act, channelwise, act_scales); + auto input = std::get<0>(quant_act); + auto input_scales = std::get<1>(quant_act); + int64_t N = weight.dim() == 4 ? weight.size(0) * weight.size(-1) : weight.size(0); + int act_quant_mode = input_scales.numel() == 1 ? PER_TENSOR + : input_scales.numel() == input.numel() / input.size(-1) ? PER_ROW + : PER_GROUP; + int wei_quant_mode = weight_scales.numel() == 1 ? PER_TENSOR : weight_scales.numel() == N ? PER_ROW : PER_GROUP; + // Case to fall back + if (weight.dim() == 2) { + TORCH_CHECK( + act_quant_mode != PER_GROUP && wei_quant_mode != PER_GROUP, + "FP8 linear: Per-group quantization is not supported in the fallback path"); + auto y_fp32 = at::linear(input.to(at::kFloat).mul_(input_scales), weight.to(at::kFloat).mul_(weight_scales), bias); + return y_fp32.to(output_dtype); + } + + static bool cpublas_can_pack = cpublas_could_pack(); + auto out_sizes = input.sizes().vec(); + out_sizes.back() = N; + auto output = at::empty(out_sizes, input.options().dtype(output_dtype)); + +#define AT_DISPATCH_FP8_LINEAR_KERNEL(OUT_DTYPE, CAN_PACK, A_QUANT_MODE, B_QUANT_MODE, ...) \ + AT_DISPATCH_BOOL_NO_RETURN( \ + CAN_PACK, \ + "cpublas_can_pack", \ + can_pack, \ + AT_DISPATCH_QUANT_MODE_NO_RETURN( \ + A_QUANT_MODE, \ + "act_quant_mode", \ + a_quant_mode, \ + AT_DISPATCH_QUANT_MODE_NO_RETURN( \ + B_QUANT_MODE, \ + "wei_quant_mode", \ + b_quant_mode, \ + AT_DISPATCH_OUT_TYPES(OUT_DTYPE, "out_dtype", __VA_ARGS__)))) + + AT_DISPATCH_FP8_LINEAR_KERNEL(output_dtype, cpublas_can_pack, act_quant_mode, wei_quant_mode, [&]() { + _float8_linear_impl( + input, input_scales, weight, weight_scales, bias, output); + }); + return output; +} + +at::Tensor float8_linear_impl( + const at::Tensor& input, + const at::Tensor& input_scales, + const at::Tensor& weight, + const at::Tensor& weight_scales, + const std::optional& bias, + at::ScalarType output_dtype) { + int64_t N = weight.dim() == 4 ? weight.size(0) * weight.size(-1) : weight.size(0); + int act_quant_mode = input_scales.numel() == 1 ? PER_TENSOR + : input_scales.numel() == input.numel() / input.size(-1) ? PER_ROW + : PER_GROUP; + int wei_quant_mode = weight_scales.numel() == 1 ? PER_TENSOR : weight_scales.numel() == N ? PER_ROW : PER_GROUP; + // Case to fall back + if (weight.dim() == 2) { + TORCH_CHECK( + act_quant_mode != PER_GROUP && wei_quant_mode != PER_GROUP, + "FP8 linear: Per-group quantization is not supported in the fallback path"); + auto y_fp32 = at::linear(input.to(at::kFloat).mul_(input_scales), weight.to(at::kFloat).mul_(weight_scales), bias); + return y_fp32.to(output_dtype); + } + + static bool cpublas_can_pack = cpublas_could_pack(); + auto out_sizes = input.sizes().vec(); + out_sizes.back() = N; + auto output = at::empty(out_sizes, input.options().dtype(output_dtype)); + +#define AT_DISPATCH_FP8_LINEAR_KERNEL(OUT_DTYPE, CAN_PACK, A_QUANT_MODE, B_QUANT_MODE, ...) \ + AT_DISPATCH_BOOL_NO_RETURN( \ + CAN_PACK, \ + "cpublas_can_pack", \ + can_pack, \ + AT_DISPATCH_QUANT_MODE_NO_RETURN( \ + A_QUANT_MODE, \ + "act_quant_mode", \ + a_quant_mode, \ + AT_DISPATCH_QUANT_MODE_NO_RETURN( \ + B_QUANT_MODE, \ + "wei_quant_mode", \ + b_quant_mode, \ + AT_DISPATCH_OUT_TYPES(OUT_DTYPE, "out_dtype", __VA_ARGS__)))) + + AT_DISPATCH_FP8_LINEAR_KERNEL(output_dtype, cpublas_can_pack, act_quant_mode, wei_quant_mode, [&]() { + _float8_linear_impl( + input, input_scales, weight, weight_scales, bias, output); + }); + return output; +} \ No newline at end of file diff --git a/csrc/cpu/sgl-kernels/moe.cpp b/csrc/cpu/sgl-kernels/moe.cpp index 3afdc1bd67..09a170eb6d 100644 --- a/csrc/cpu/sgl-kernels/moe.cpp +++ b/csrc/cpu/sgl-kernels/moe.cpp @@ -836,6 +836,11 @@ static inline void check_moe_scales( TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4."); TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8."); TORCH_CHECK(w2_scale.value().scalar_type() == at::kByte, "expect w2_scale to be uint8."); + } else if constexpr (quant == CPUQuantMethod::FP8_W8A8) { + TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a8."); + TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a8."); + TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a8."); + TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2."); } } @@ -850,9 +855,14 @@ static inline void check_moe_scales( check_moe_scales(w1_scale, w2_scale, block_size); } else if (moe_comp_method == CPUQuantMethod::MXFP4) { check_moe_scales(w1_scale, w2_scale, block_size); + } else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + check_moe_scales(w1_scale, w2_scale, block_size); } } +// CHECK_MOE_SCALES_FP8: validate scale shapes for FP8_W8A16. +// w1s shape [E, 2N/block_N, K/block_K, block_N]: DIM0=1, DIM1=2 +// w2s shape [E, K/block_N, N/block_K]: DIM0=1, DIM1=2 #define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \ auto w1s = w1_scale.value(); \ auto w2s = w2_scale.value(); \ @@ -864,6 +874,17 @@ static inline void check_moe_scales( TORCH_CHECK(w2s.size(DIM0) == div_up(K, block_size_N)); \ TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K)) +// CHECK_MOE_SCALES_FP8_W8A8: extract scale info for FP8_W8A8 without shape +// validation, because w1s is pre-packed by float8_linear_prepack_cpu with +// shape [E, 2N/BLOCK_N, G, BLOCK_N] where BLOCK_N=32 != block_size_N(128). +// This matches the sglang upstream behavior (checks are commented out there). +#define CHECK_MOE_SCALES_FP8_W8A8(DIM0, DIM1) \ + auto w1s = w1_scale.value(); \ + auto w2s = w2_scale.value(); \ + auto block_size_val = block_size.value(); \ + int64_t block_size_N = block_size_val[0]; \ + int64_t block_size_K = block_size_val[1] + // hidden_states: [M, K] // w1: [E, 2N, K] or [E, 2N, K / 2] for uint8 // w2: [E, K, N] or [E, K, N / 2] for uint8 @@ -883,6 +904,7 @@ void fused_experts_cpu( const std::optional& w2_scale, const std::optional& w1_zero, const std::optional& w2_zero, + const std::optional& a1_scale, const std::optional> block_size, const std::optional& w1_bias, const std::optional& w2_bias, @@ -895,7 +917,10 @@ void fused_experts_cpu( constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); - const auto st = hidden_states.scalar_type(); + // FP8_W8A8 hidden_states input is FP8; scalar_t should be BF16 for output + const auto st = (moe_comp_method == CPUQuantMethod::FP8_W8A8) + ? at::kBFloat16 + : hidden_states.scalar_type(); // TODO: fused_topk_torch_native (CPU fallback for models like MiniMax) // returns int64 topk_ids; fused_experts_cpu requires int32. Remove the typecast after topk kernel is provided auto topk_ids_ = topk_ids.scalar_type() == at::kInt ? topk_ids : topk_ids.to(at::kInt); @@ -908,6 +933,9 @@ void fused_experts_cpu( if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) { CHECK_DIM(4, w1); CHECK_DIM(4, w2); + } else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + // w1 is prepackaged by float8_linear_prepack_cpu (5D), skip w1 dim check + CHECK_DIM(3, w2); } else { CHECK_DIM(3, w1); CHECK_DIM(3, w2); @@ -924,8 +952,9 @@ void fused_experts_cpu( int64_t M = hidden_states.size(0); int64_t K = hidden_states.size(1); - int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8 ? w1_scale.value().size(1) * w1_scale.value().size(3) / 2 - : w1.size(1) / 2; + int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8 + ? w1_scale.value().size(1) * w1_scale.value().size(3) / 2 + : (moe_comp_method == CPUQuantMethod::FP8_W8A8 ? w2.size(2) : w1.size(1) / 2); int64_t E = w1.size(0); int64_t topk = topk_weights_.size(1); @@ -935,7 +964,10 @@ void fused_experts_cpu( // check weight shapes CHECK_EQ(w2.size(0), E); - if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) { + if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + CHECK_EQ(w2.size(1), K); + CHECK_EQ(packed_w2.size(2), packed_N); + } else if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) { CHECK_EQ(w2.size(1), K); CHECK_EQ(packed_w1.size(2), packed_K / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1)); CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1)); @@ -993,22 +1025,35 @@ void fused_experts_cpu( // 5. Aq_tmp : [M, K] or [M * topk, N] // 6. As_tmp : [M * topk] // - // for fp8 w8a16 and mxfp4: + // for fp8 w8a16, mxfp4, and fp8 w8a8: // 7. intermediate_cache0 : [M * topk, 2N] // 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)] // + // for fp8 w8a8 additionally: + // 9. Ukernel_tmp : [T, 2 * BLOCK_M * BLOCK_N] (float32 for FP8×FP8 brgemm) + // int64_t buffer_size_nbytes = M * topk * N * 2 + M * topk * K * 2 + num_threads * BLOCK_M * K * - (moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 1 : 2) + + (moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8 + | moe_comp_method == CPUQuantMethod::FP8_W8A8 ? 1 : 2) + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); if (moe_comp_method == CPUQuantMethod::INT8_W8A8) { buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float); } - if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4) { + if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4 + || moe_comp_method == CPUQuantMethod::FP8_W8A8) { buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2; } + if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + // Ukernel buffer: float32 workspace per thread for FP8×FP8 brgemm + buffer_size_nbytes += num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); +#ifndef CPUBLAS_BRGEMM_F8F8F32 + // BF16 fallback: also need dqA [T, BLOCK_M, BLOCK_K] + dqB [T, BLOCK_K, BLOCK_N] + buffer_size_nbytes += num_threads * (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N) * 2; +#endif + } if (moe_comp_method == CPUQuantMethod::INT4_W4A8) { buffer_size_nbytes += M * topk * 2 * N * 2 + std::max(M * K, M * topk * N) + M * topk * sizeof(float) + num_threads * 2 * get_4bit_block_k_size(K / w1_scale.value().size(2)) * BLOCK_N; @@ -1181,6 +1226,47 @@ void fused_experts_cpu( E, topk, num_tokens_post_pad); + } else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + // Stage 1: FP8×FP8 GEMM (hidden_states is FP8 input) + // Stage 2: BF16×FP8 GEMM (W8A16 path for down-projection) + at::Float8_e4m3fn* __restrict__ A_tmp = + (at::Float8_e4m3fn*)((void*)(intermediate_cache2 + M * topk * K)); + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + scalar_t* __restrict__ intermediate_cache0 = + (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + scalar_t* __restrict__ B_tmp = + (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N)); + float* __restrict__ Ukernel_tmp = + (float*)((void*)(B_tmp + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N))); + + CHECK_MOE_SCALES_FP8_W8A8(1, 2); + fused_experts_fp8_a8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache0, + intermediate_cache1, + intermediate_cache2, + A_tmp, + B_tmp, + C_tmp, + Ukernel_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + a1_scale.has_value() ? a1_scale.value().data_ptr() : nullptr, + w1s.data_ptr(), + w2s.data_ptr(), + block_size_N, + block_size_K, + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); } else { scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K; float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); diff --git a/csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp b/csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp new file mode 100644 index 0000000000..0f393a34a8 --- /dev/null +++ b/csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp @@ -0,0 +1,432 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +#include + +#include "common.h" +#include "gemm.h" +#include "moe.h" + +namespace { + +// Byte-wise copy for FP8 elements (no remainder assumption) +inline void copy_stub2( + at::Float8_e4m3fn* __restrict__ out, + const at::Float8_e4m3fn* __restrict__ input, + int64_t size) { + for (int64_t d = 0; d < size; d++) { + out[d] = input[d]; + } +} + +#define PER_TENSOR 1 +#define PER_ROW 2 +#define PER_GROUP 3 + +// Store float32 GEMM result to output buffer with dequantization and dtype +// conversion. Activations are PER_ROW quantized; weights are PER_GROUP quantized. +// Uses AVX512 on capable platforms. +#if defined(CPU_CAPABILITY_AVX512) + +template +inline void store_out( + const float* y_buf, + out_dtype* c_ptr, + int64_t M, + int64_t lda, + const float* scales_a, + const float* scales_b, + const float* bias) { + float a_scale = 1.0, b_scale = 1.0; + __m512 va_scale, vb_scale; + if constexpr (act_quant_mode == PER_TENSOR) { + a_scale = *scales_a; + } + if constexpr (wei_quant_mode == PER_TENSOR) { + b_scale = *scales_b; + vb_scale = _mm512_set1_ps(b_scale); + } + for (int i = 0; i < M; ++i) { + if constexpr (act_quant_mode == PER_ROW) { + a_scale = *(scales_a + i); + } + if constexpr (act_quant_mode != PER_GROUP) { + va_scale = _mm512_set1_ps(a_scale); + } + constexpr int N_UNROLL = N / 16; + c10::ForcedUnroll{}([&](auto idx) { + constexpr int j = idx * 16; + __m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j); + __m512 bias_vec = bias ? _mm512_loadu_ps(bias + j) : _mm512_setzero_ps(); + if constexpr (act_quant_mode != PER_GROUP) { + y_vec = _mm512_mul_ps(y_vec, va_scale); + } + if constexpr (wei_quant_mode == PER_ROW) { + vb_scale = _mm512_loadu_ps(scales_b + j); + } + if constexpr (wei_quant_mode != PER_GROUP) { + y_vec = _mm512_mul_ps(y_vec, vb_scale); + } + y_vec = _mm512_add_ps(y_vec, bias_vec); + if constexpr (std::is_same::value) { + _mm512_storeu_ps(c_ptr + i * lda + j, y_vec); + } else if constexpr (std::is_same::value) { + __m256i y_bf16_vec = at::vec::cvtfp32_bf16(y_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_bf16_vec); + } else if constexpr (std::is_same::value) { + __m256i y_fp16_vec = at::vec::cvtfp32_fp16(y_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_fp16_vec); + } else { + TORCH_CHECK(false, "Unsupported output dtype"); + } + }); + constexpr int tail_start = N / 16 * 16; + for (int j = tail_start; j < N; ++j) { + if constexpr (wei_quant_mode == PER_ROW) { + b_scale = scales_b[j]; + } + c_ptr[i * lda + j] = static_cast(y_buf[i * N + j] * a_scale * b_scale); + } + } // for M +} + +#else // no AVX512 + +template +inline void store_out( + const float* y_buf, + out_dtype* c_ptr, + int64_t M, + int64_t lda, + const float* scales_a, + const float* scales_b, + const float* bias) { + float a_scale = 1.0, b_scale = 1.0; + if constexpr (act_quant_mode == PER_TENSOR) { + a_scale = *scales_a; + } + if constexpr (wei_quant_mode == PER_TENSOR) { + b_scale = *scales_b; + } + for (int i = 0; i < M; ++i) { + if constexpr (act_quant_mode == PER_ROW) { + a_scale = *(scales_a + i); + } + for (int j = 0; j < N; ++j) { + if constexpr (wei_quant_mode == PER_ROW) { + b_scale = scales_b[j]; + } + float val = y_buf[i * N + j] * a_scale * b_scale; + if (bias) val += bias[j]; + c_ptr[i * lda + j] = static_cast(val); + } + } +} + +#endif // CPU_CAPABILITY_AVX512 + +} // anonymous namespace + +// FP8 W8A8 fused MoE kernel. +// +// hidden_states: [M, K] FP8 E4M3, pre-quantized activation +// packed_w1: [E, 2*Nc, Kc, BLOCK_K, BLOCK_N] FP8 W8A8 prepack (float8_linear_prepack_cpu) +// packed_w2: [E, K, N_packed] FP8 W8A16 VNNI pack (convert_weight_packed) +// As: [M] float32 per-token activation scales +// w1s: [E, num_groups, 2N] float32 per-group weight scales for w1 +// w2s: [E, ...] float32 per-group weight scales for w2 +// +// Stage 1: hidden_states(FP8) @ w1(FP8) → ic0(BF16) [M*topk, 2N] +// Uses FP8×FP8 GEMM (tinygemm_kernel with FP8 inputs) +// Stage 1.5: ic1 = silu(ic0[:,:N]) * ic0[:,N:] [M*topk, N] +// Stage 2: ic1(BF16) @ w2(FP8) → ic2(BF16) [M*topk, K] +// Uses BF16×FP8 GEMM (W8A16 path, tinygemm_kernel) +// Stage 3: out = weighted_sum(ic2, topk_weights) [M, K] + +template +void fused_experts_fp8_a8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + at::Float8_e4m3fn* __restrict__ A_tmp, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + float* __restrict__ Ukernel_tmp, + const at::Float8_e4m3fn* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ As, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + // 1. intermediate_cache1 : [M * topk, N] + // 2. intermediate_cache2 : [M * topk, K] + // 3. A_tmp : [T, BLOCK_M * K] FP8 + // 4. C_tmp : [T, 2 * BLOCK_M * BLOCK_N] float32 + // 5. intermediate_cache0 : [M * topk, 2N] + // 6. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, max(K, N)] scalar_t + // 7. Ukernel_tmp: [T, 2 * BLOCK_M * BLOCK_N] float32 + // 8. (non-FP8-brgemm only) dqA [T, BLOCK_M * BLOCK_K] + dqB [T, BLOCK_K * BLOCK_N] in BF16 + // appended after Ukernel_tmp; see moe.cpp FP8_W8A8 buffer allocation + + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N_VAL = block_size_n(); // = 32 + const int64_t KB = K / BLOCK_K; // K blocks + + int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N_VAL * std::max(K, N); + + // stage 1: intermediate_cache0 = hidden_states @ w1 + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N_VAL); + + int64_t scale_size_N = div_up(2 * N, block_size_N); + int64_t scale_size_K = div_up(K, block_size_K); + int64_t blocks_n_per_group = block_size_N / BLOCK_N_VAL; + int64_t num_groups = div_up(K, block_size_K); // G + int64_t blocks_k_per_group = block_size_K / BLOCK_K; + const int64_t stride_e = 2 * N * K; // per-expert stride for w1 (total elements) + bool use_brgemm = true; + int64_t num_thread = at::get_num_threads(); + + // For the BF16 fallback path (no native FP8 brgemm), tinygemm_kernel needs temporary + // BF16 buffers to dequantize FP8 inputs before running AMX-BF16 brgemm. + // These are placed after the Ukernel_tmp region (see moe.cpp FP8_W8A8 buffer allocation). +#ifndef CPUBLAS_BRGEMM_F8F8F32 + // dqA: [T, BLOCK_M, BLOCK_K] BF16 + // dqB: [T, BLOCK_K, BLOCK_N] BF16 + // Layout: immediately after Ukernel_tmp[num_thread * 2 * BLOCK_M * BLOCK_N_VAL] + at::BFloat16* __restrict__ dq_base = reinterpret_cast( + Ukernel_tmp + num_thread * 2 * BLOCK_M * BLOCK_N_VAL); + const int64_t dq_per_thread = BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N_VAL; +#endif + + at::parallel_for(0, MB * NB, 1, [&](int64_t begin, int64_t end) { + int tid = get_thread_num(); + // A_tmp: FP8 activation buffer for this thread [BLOCK_M, K] + at::Float8_e4m3fn* __restrict__ A = A_tmp + tid * BLOCK_M * K; + float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL; + float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N_VAL; + alignas(64) float As_[BLOCK_M]; + float* __restrict__ ukernel_buf_1 = Ukernel_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL; +#ifndef CPUBLAS_BRGEMM_F8F8F32 + at::BFloat16* __restrict__ dqA_buf = dq_base + tid * dq_per_thread; + at::BFloat16* __restrict__ dqB_buf = dqA_buf + BLOCK_M * BLOCK_K; +#else + at::BFloat16* __restrict__ dqA_buf = nullptr; + at::BFloat16* __restrict__ dqB_buf = nullptr; +#endif + float* __restrict__ ukernel_buf_2 = ukernel_buf_1 + BLOCK_M * BLOCK_N_VAL; + auto ldsa = 1; // activation per-row quantized + + for (int64_t i = begin; i < end; ++i) { + int64_t mb = i / NB; + int64_t nb = i % NB; + int64_t nb1 = nb + NB; // gate is first N columns, up is second N columns + + int64_t n_size = std::min(N - nb * BLOCK_N_VAL, BLOCK_N_VAL); + + // B (weight) for expert expert_id: gate+up fused [2N, K] prepackaged as FP8 + int32_t expert_id = expert_ids[mb]; + const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e; + const float* __restrict__ Bs = w1s + expert_id * num_groups * (2 * N); + + // Load activation tile for this M-block + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub2(A + m * K, input + index * K, K); + As_[m] = As[index]; + } + + const int64_t offset = offsets[mb]; + + // Gate path (nb): w1[:N, :] + zero_buffer(C0, BLOCK_M * BLOCK_N_VAL); + for (int kci = 0; kci < KB; ++kci) { + // FP8×FP8 tinygemm: act_quant=PER_ROW(2), wei_quant=PER_GROUP(3) + tinygemm_kernel( + /* C */ C0, + /* A */ A + kci * BLOCK_K, + /* scales_a*/ As_ + kci / blocks_k_per_group, + /* B */ B + (nb * KB + kci) * BLOCK_K * BLOCK_N_VAL, + /* scales_b*/ Bs + nb * BLOCK_N_VAL * num_groups + kci / blocks_k_per_group * BLOCK_N_VAL, + /* M */ m_size, + /* K */ BLOCK_K, + /* lda */ K, + /* ldc */ BLOCK_N_VAL, + /* ldsa */ ldsa, + /* ukernel */ ukernel_buf_1, + /* dqA_buf */ dqA_buf, + /* dqB_buf */ dqB_buf); + } + store_out( + C0, + ic0 + offset * 2 * N + nb * BLOCK_N_VAL, + m_size, + 2 * N, // lda + As_, + nullptr, + nullptr); + + // Up path (nb1): w1[N:, :] + zero_buffer(C1, BLOCK_M * BLOCK_N_VAL); + for (int kci = 0; kci < KB; ++kci) { + tinygemm_kernel( + /* C */ C1, + /* A */ A + kci * BLOCK_K, + /* scales_a*/ As_ + kci / blocks_k_per_group, + /* B */ B + (nb1 * KB + kci) * BLOCK_K * BLOCK_N_VAL, + /* scales_b*/ Bs + nb1 * BLOCK_N_VAL * num_groups + kci / blocks_k_per_group * BLOCK_N_VAL, + /* M */ m_size, + /* K */ BLOCK_K, + /* lda */ K, + /* ldc */ BLOCK_N_VAL, + /* ldsa */ ldsa, + /* ukernel */ ukernel_buf_2, + /* dqA_buf */ dqA_buf, + /* dqB_buf */ dqB_buf); + } + store_out( + C1, + ic0 + offset * 2 * N + nb1 * BLOCK_N_VAL, + m_size, + 2 * N, // lda + As_, + nullptr, + nullptr); + } + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: intermediate_cache1 = silu(ic0[:, :N]) * ic0[:, N:] + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2: [E, K, N] FP8, VNNI packed; treated as W8A16 (BF16 act × FP8 weight) + const int64_t OC = K; // output channels (hidden dim) + const int64_t IC = N; // input channels (expert intermediate dim) + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N_VAL); + scale_size_N = div_up(K, block_size_N); + scale_size_K = div_up(N, block_size_K); + const int64_t stride_e2 = OC * IC; // per-expert stride for w2 + const int64_t stride_oc = IC; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + bool use_brgemm2 = can_use_brgemm(avg_M); + + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N_VAL * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N_VAL, BLOCK_N_VAL); + + // A from ic1: [M*topk, N] in sorted order (contiguous after stage 1.5) + const scalar_t* __restrict__ A2 = ic1 + offsets[mb] * N; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B from w2: [E, K, N] VNNI packed, offset to expert + output block + int32_t expert_id = expert_ids[mb]; + const at::Float8_e4m3fn* __restrict__ B2 = + packed_w2 + expert_id * stride_e2 + nb * BLOCK_N_VAL * stride_oc; + const float* __restrict__ Bs2 = + w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + + // Only unpack B on first visit or new expert + int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; + bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); + + tinygemm_kernel( + /* A */ A2, + /* B */ B2, + /* C */ C, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N_VAL * IC, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL, + /* Bbias */ nullptr, + /* scale */ Bs2, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N_VAL, + /* brg */ use_brgemm2, + /* block_sz_K */ block_size_K, + /* do_unpack */ do_unpack); + + // Scatter ic2 in original token order, weighting by topk_weights + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N_VAL, C + m * BLOCK_N_VAL, weight, n_size); + } + }); + + if (use_brgemm2) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: output = sum(ic2, dim=topk) [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +#define INSTANTIATE_MOE_FP8_A8_TEMPLATE(TYPE) \ + template void fused_experts_fp8_a8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic0, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ ic2, \ + at::Float8_e4m3fn* __restrict__ A_tmp, \ + TYPE* __restrict__ B_tmp, \ + float* __restrict__ C_tmp, \ + float* __restrict__ Ukernel_tmp, \ + const at::Float8_e4m3fn* __restrict__ input, \ + const at::Float8_e4m3fn* __restrict__ packed_w1, \ + const at::Float8_e4m3fn* __restrict__ packed_w2, \ + const float* __restrict__ As, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + int64_t block_size_N, \ + int64_t block_size_K, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad) + +INSTANTIATE_MOE_FP8_A8_TEMPLATE(at::BFloat16); +INSTANTIATE_MOE_FP8_A8_TEMPLATE(at::Half); diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index 88170068de..accd5fcec3 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -73,6 +73,7 @@ void fused_experts_cpu(at::Tensor& out, at::Tensor& hidden_states, const std::optional& w2_scale, const std::optional& w1_zero, const std::optional& w2_zero, + const std::optional& a1_scale, const std::optional> block_size, const std::optional& w1_bias, const std::optional& w2_bias, @@ -91,6 +92,18 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, at::ScalarType out_dtype, bool is_vnni); +// Adapted from sglang: FP8 W8A8 kernels +std::tuple float8_linear_prepack_impl( + const at::Tensor& weight, const at::Tensor& scales); +at::Tensor fp8_scaled_mm_with_quant(const at::Tensor& act, + const std::optional& act_scales, + bool channelwise, const at::Tensor& weight, + const at::Tensor& weight_scales, + const std::optional& bias, + at::ScalarType output_dtype); +std::tuple quantize_fp8e4m3_vec( + const at::Tensor& t, bool channelwise, c10::optional scale_opt); + // Adapted from sglang: MLA CPU kernels (AMX-only) void decode_attention_cpu(at::Tensor& query, at::Tensor& k_buffer, at::Tensor& v_buffer, at::Tensor& output, @@ -281,6 +294,7 @@ void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input, const std::string& activation); bool cpu_attn_has_isa(const std::string& isa); +bool cpu_has_amx_fp8(); torch::Tensor get_scheduler_metadata( const int64_t num_req, const int64_t num_heads_q, @@ -611,7 +625,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "fused_experts_cpu(Tensor(a0!) out, Tensor hidden_states, Tensor w1, " "Tensor w2, Tensor topk_weights, Tensor topk_ids, " "int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, " - "Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, " + "Tensor? w1_zero, Tensor? w2_zero, Tensor? a1_scale, int[]? block_size, " "Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, " "bool is_vnni) -> " "()"); @@ -629,7 +643,23 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "bool is_vnni) -> Tensor"); ops.impl("fp8_scaled_mm_cpu", torch::kCPU, &fp8_scaled_mm_cpu); - // Adapted from sglang: casual_conv1d kernels + // Adapted from sglang: FP8 W8A8 kernels + ops.def( + "float8_linear_prepack_cpu(Tensor weight, Tensor scales) -> (Tensor, " + "Tensor)"); + ops.impl("float8_linear_prepack_cpu", torch::kCPU, + &float8_linear_prepack_impl); + ops.def( + "fp8_scaled_mm_with_quant(Tensor act, Tensor? act_scales, bool " + "channelwise, " + "Tensor weight, Tensor weight_scales, Tensor? bias, ScalarType " + "output_dtype) -> Tensor"); + ops.impl("fp8_scaled_mm_with_quant", torch::kCPU, &fp8_scaled_mm_with_quant); + ops.def( + "quantize_fp8e4m3_vec(Tensor input, bool channelwise, Tensor? scale_opt) " + "-> (Tensor, Tensor)"); + ops.impl("quantize_fp8e4m3_vec", torch::kCPU, &quantize_fp8e4m3_vec); + ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor"); ops.impl("causal_conv1d_weight_pack", torch::kCPU, &causal_conv1d_weight_pack); @@ -832,6 +862,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "-> (Tensor, Tensor)"); ops.impl("fused_gdn_gating_cpu", torch::kCPU, &fused_gdn_gating_cpu); + // Pure runtime CPU ISA capability check. + ops.def("cpu_has_amx_fp8() -> bool", &cpu_has_amx_fp8); + // CPU attention kernels ops.def("cpu_attn_has_isa(str isa) -> bool", &cpu_attn_has_isa); ops.def( diff --git a/tests/kernels/moe/test_cpu_quant_fused_moe.py b/tests/kernels/moe/test_cpu_quant_fused_moe.py index 99cae4534b..e89fbb55a0 100644 --- a/tests/kernels/moe/test_cpu_quant_fused_moe.py +++ b/tests/kernels/moe/test_cpu_quant_fused_moe.py @@ -1114,5 +1114,203 @@ def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace): ) +# =========================================================================== +# FP8 W8A8 MoE +# =========================================================================== + + +requires_cpu_fp8_w8a8 = pytest.mark.skipif( + not ops.cpu_has_amx_fp8(), + reason="requires native AMX-FP8 hardware", +) + +FP8_W8A8_MAX = torch.finfo(torch.float8_e4m3fn).max +FP8_W8A8_QUANT_GROUP = 128 # quantization group size for both K and N dimensions + + +def _make_fp8_w8a8_weight_w13( + E: int, two_n: int, K: int, group_K: int = FP8_W8A8_QUANT_GROUP +) -> tuple[torch.Tensor, torch.Tensor]: + """Create w13 [E, 2N, K] FP8 with group scales [E, 2N, G] where G = K//group_K.""" + G = K // group_K + w_list, s_list = [], [] + for _ in range(E): + w_f32 = torch.randn(two_n, K) + # Per-row, per-K-group quantization: scale [2N, G] + w_re = w_f32.view(two_n, G, group_K) + abs_max = w_re.abs().amax(dim=2, keepdim=True).clamp(min=1e-7) # [2N, G, 1] + scale = (abs_max / FP8_W8A8_MAX).squeeze(2) # [2N, G] + w_q = ( + (w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn) + ) + w_list.append(w_q.view(two_n, K).contiguous()) + s_list.append(scale.float()) + return torch.stack(w_list), torch.stack(s_list) # [E, 2N, K], [E, 2N, G] + + +def _make_fp8_w8a8_weight_w2( + E: int, + K: int, + N: int, + group_K: int = FP8_W8A8_QUANT_GROUP, + group_N: int = FP8_W8A8_QUANT_GROUP, +) -> tuple[torch.Tensor, torch.Tensor]: + """Create w2 [E, K, N] FP8 with block scales [E, K//gK, N//gN].""" + nK = K // group_K + nN = N // group_N + w_list, s_list = [], [] + for _ in range(E): + w_f32 = torch.randn(K, N) + w_re = w_f32.view(nK, group_K, nN, group_N) + abs_max = w_re.abs().amax(dim=(1, 3), keepdim=True).clamp(min=1e-7) + scale = (abs_max / FP8_W8A8_MAX).squeeze(1).squeeze(2) # [nK, nN] + w_q = ( + (w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn) + ) + w_list.append(w_q.view(K, N).contiguous()) + s_list.append(scale.float()) + return torch.stack(w_list), torch.stack(s_list) # [E, K, N], [E, K//gK, N//gN] + + +def _pack_fp8_w8a8_w13( + w13: torch.Tensor, # [E, 2N, K] FP8 + w13_scale: torch.Tensor, # [E, 2N, G] float32 +) -> tuple[torch.Tensor, torch.Tensor]: + """Pack w13 for CPU FP8 W8A8 MoE kernel using float8_linear_prepack_cpu.""" + E = w13.size(0) + packed_list, scale_list = [], [] + for i in range(E): + pw, ps = torch.ops._C.float8_linear_prepack_cpu( + w13[i].contiguous(), w13_scale[i].contiguous() + ) + packed_list.append(pw) + scale_list.append(ps) + return torch.stack(packed_list), torch.stack(scale_list) + + +def _ref_fp8_w8a8_moe( + hidden_states: torch.Tensor, # BF16 [M, K_hidden] (after FP8 round-trip dequant) + w13: torch.Tensor, # FP8 [E, 2N, K_hidden] + w13_scale: torch.Tensor, # float32 [E, 2N, G] (G = K_hidden//group_K) + w2: torch.Tensor, # FP8 [E, K_hidden, N] + w2_scale: torch.Tensor, # float32 [E, K_hidden//gK, N//gN] + topk_weights: torch.Tensor, # float32 [M, top_k] + topk_ids: torch.Tensor, # int64 [M, top_k] + N: int, + group_K: int = FP8_W8A8_QUANT_GROUP, + group_N: int = FP8_W8A8_QUANT_GROUP, +) -> torch.Tensor: + """Reference implementation: dequantize weights then compute MoE.""" + M, K_hidden = hidden_states.shape + top_k = topk_ids.shape[1] + output = torch.zeros(M, K_hidden, dtype=torch.bfloat16) + + for tok in range(M): + x = hidden_states[tok].float() # [K_hidden] + for k_idx in range(top_k): + expert_id = topk_ids[tok, k_idx].item() + w = topk_weights[tok, k_idx].item() + + # Stage 1: x @ w13.T — gate and up projections + w13_e = w13[expert_id].float() # [2N, K_hidden] + ws_e = w13_scale[expert_id].float() # [2N, G] + G = ws_e.shape[1] + w13_dq = torch.zeros_like(w13_e) + for g in range(G): + c0, c1 = g * group_K, (g + 1) * group_K + w13_dq[:, c0:c1] = w13_e[:, c0:c1] * ws_e[:, g : g + 1] + gate_up = (x.unsqueeze(0) @ w13_dq.T).squeeze(0) # [2N] + + gate = gate_up[:N] + up = gate_up[N:] + act = F.silu(gate) * up # [N] + + # Stage 2: act @ w2.T — down projection + w2_e = w2[expert_id].float() # [K_hidden, N] + ws2_e = w2_scale[expert_id].float() # [K_hidden//gK, N//gN] + nK2 = K_hidden // group_K + nN2 = N // group_N + w2_dq = torch.zeros_like(w2_e) + for gi in range(nK2): + for gj in range(nN2): + r0, r1 = gi * group_K, (gi + 1) * group_K + c0, c1 = gj * group_N, (gj + 1) * group_N + w2_dq[r0:r1, c0:c1] = w2_e[r0:r1, c0:c1] * ws2_e[gi, gj] + out_e = (act.unsqueeze(0) @ w2_dq.T).squeeze(0) # [K_hidden] + + output[tok] += (w * out_e).to(torch.bfloat16) + + return output + + +@requires_cpu_fp8_w8a8 +@pytest.mark.parametrize( + "E,N,K,M,top_k", + [ + # N and K must be multiples of FP8_W8A8_QUANT_GROUP=128 + (4, 128, 256, 1, 2), # single token (decode phase) + (4, 128, 256, 8, 2), + (8, 256, 512, 16, 2), + (4, 256, 512, 4, 1), + ], +) +def test_fp8_w8a8_cpu_fused_moe(E: int, N: int, K: int, M: int, top_k: int): + """Test fused_experts_cpu FP8_W8A8 shape/dtype and accuracy vs a + dequantized BF16 reference.""" + set_random_seed(42) + + w13, w13_scale = _make_fp8_w8a8_weight_w13(E, 2 * N, K) + w2, w2_scale = _make_fp8_w8a8_weight_w2(E, K, N) + + packed_w13, packed_w13_scale = _pack_fp8_w8a8_w13(w13, w13_scale) + packed_w2 = _prepack_experts(w2) + + hidden_states = torch.randn(M, K, dtype=torch.bfloat16) * 0.1 + x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec(hidden_states, True, None) + assert x_fp8.dtype == torch.float8_e4m3fn + assert x_scales.shape == (M,) + + gen = torch.Generator().manual_seed(0) + topk_weights = torch.softmax(torch.randn(M, top_k, generator=gen), dim=-1).float() + topk_ids = torch.zeros(M, top_k, dtype=torch.int32) + for i in range(M): + topk_ids[i] = torch.randperm(E, generator=torch.Generator().manual_seed(i))[ + :top_k + ].int() + + output = torch.empty(M, K, dtype=torch.bfloat16) + ops.fused_experts_cpu( + output, + x_fp8, + packed_w13, + packed_w2, + topk_weights, + topk_ids, + ops.CPUQuantMethod.FP8_W8A8, + packed_w13_scale, # w1_scale: [E, Nc, G, BLOCK_N] + w2_scale, # w2_scale: [E, K//128, N//128] + None, # w1_zero + None, # w2_zero + [FP8_W8A8_QUANT_GROUP, FP8_W8A8_QUANT_GROUP], # block_size + None, # w1_bias + None, # w2_bias + None, # alpha + None, # limit + True, # is_vnni (weights already packed) + x_scales, # a1_scale: per-token FP8 activation scales + ) + + assert output.shape == (M, K) + assert output.dtype == torch.bfloat16 + assert not torch.isnan(output).any() + assert not torch.isinf(output).any() + + x_dequant = (x_fp8.float() * x_scales.unsqueeze(1)).bfloat16() + ref = _ref_fp8_w8a8_moe( + x_dequant, w13, w13_scale, w2, w2_scale, topk_weights, topk_ids.long(), N + ) + torch.testing.assert_close(output.float(), ref.float(), atol=1e-2, rtol=1e-2) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py b/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py index e4dadb53bd..0b0d12a66c 100644 --- a/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py +++ b/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py @@ -1,6 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for CPU FP8 W8A16 block-scaled GEMM kernel (fp8_scaled_mm_cpu). +"""Tests for CPU FP8 scaled-mm GEMM kernels. + +Covers: + * fp8_scaled_mm_cpu (W8A16) + * fp8_scaled_mm_with_quant (W8A8) Run `pytest tests/kernels/quantization/test_cpu_fp8_scaled_mm.py -v`. """ @@ -23,19 +27,31 @@ from vllm.platforms import current_platform if not current_platform.is_cpu(): pytest.skip("skipping CPU-only tests", allow_module_level=True) -if not ops._supports_cpu_fp8_w8a16: - pytest.skip("fp8_scaled_mm_cpu op not available", allow_module_level=True) +requires_cpu_fp8_w8a16 = pytest.mark.skipif( + not ops._supports_cpu_fp8_w8a16, reason="fp8_scaled_mm_cpu op not available" +) +requires_cpu_fp8_w8a8 = pytest.mark.skipif( + not ops.cpu_has_amx_fp8(), + reason="requires native AMX-FP8 hardware", +) if not torch.cpu._is_amx_tile_supported(): pytest.skip("requires AMX tile support", allow_module_level=True) +FP8_MAX = torch.finfo(torch.float8_e4m3fn).max BLOCK_SIZE = [128, 128] +GROUP_SIZE = 128 def cdiv(a: int, b: int) -> int: return -(a // -b) +# --------------------------------------------------------------------------- +# W8A16 helpers +# --------------------------------------------------------------------------- + + def quantize_weight_block_fp8( weight: torch.Tensor, block_size: list[int], @@ -126,7 +142,7 @@ def ref_fp8_block_scaled_mm( # --------------------------------------------------------------------------- -# Test parameters +# W8A16 test: fp8_scaled_mm_cpu # --------------------------------------------------------------------------- M_SIZES = [1, 4, 16, 64, 128] # (N, K) — weight shape is [N, K], output has N columns. @@ -141,6 +157,7 @@ NK_SIZES = [ ] +@requires_cpu_fp8_w8a16 @pytest.mark.parametrize("M", M_SIZES) @pytest.mark.parametrize("N,K", NK_SIZES) @pytest.mark.parametrize("use_bias", [False, True]) @@ -200,6 +217,77 @@ def ref_fp8_per_tensor_scaled_mm( return out.to(out_dtype) +# --------------------------------------------------------------------------- +# W8A8 helpers +# --------------------------------------------------------------------------- + + +def quantize_weight_per_tensor( + weight: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Quantize weight [N, K] to FP8 with a single per-tensor scale, broadcast + to [N, 1] (the shape float8_linear_prepack_cpu expects).""" + N = weight.shape[0] + abs_max = weight.abs().max() + scale = (abs_max / FP8_MAX).clamp(min=1e-7) + q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q, scale.expand(N).contiguous().view(N, 1) + + +def quantize_weight_per_channel( + weight: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Quantize weight [N, K] to FP8 with per-channel (per-row) scales [N, 1].""" + abs_max = weight.abs().amax(dim=1, keepdim=True) + scale = (abs_max / FP8_MAX).clamp(min=1e-7) + q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q, scale.float() + + +def quantize_weight_per_group( + weight: torch.Tensor, + group_size: int = GROUP_SIZE, +) -> tuple[torch.Tensor, torch.Tensor]: + """Quantize weight [N, K] to FP8 with per-group (128-block) scales [N, G].""" + N, K = weight.shape + G = K // group_size + w_groups = weight.view(N, G, group_size) + abs_max = w_groups.abs().amax(dim=2, keepdim=True) + scale = (abs_max / FP8_MAX).clamp(min=1e-7) + q = (w_groups.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q.view(N, K), scale.squeeze(2).float() + + +def quantize_act_per_token(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Reference per-token (row) dynamic FP8 quantization of activations.""" + x_f = x.float() + scale = (x_f.abs().amax(dim=1, keepdim=True) / FP8_MAX).clamp(min=1e-7) + scaled = (x_f / scale).clamp(-FP8_MAX, FP8_MAX) + # Round via fp16 to match the AVX10.2 kernel path. + q = scaled.half().to(torch.float8_e4m3fn) + return q, scale + + +def reference_static_fp8_linear( + x: torch.Tensor, # BF16 [M, K] + weight_fp8: torch.Tensor, # FP8 [N, K] + weight_scale: torch.Tensor, # float32 [N, 1] + act_scale: torch.Tensor, # float32 scalar + bias: torch.Tensor | None, + out_dtype: torch.dtype, +) -> torch.Tensor: + """Static per-tensor: quantize/dequant x with act_scale, dequant weight, matmul.""" + x_scaled = (x.float() / act_scale.item()).clamp(-FP8_MAX, FP8_MAX) + # Round via fp16 to match the AVX10.2 kernel path. + x_q = x_scaled.half().to(torch.float8_e4m3fn) + x_dq = x_q.float() * act_scale.item() + w_dq = weight_fp8.float() * weight_scale.view(-1, 1) + out = torch.mm(x_dq, w_dq.t()) + if bias is not None: + out = out + bias.float() + return out.to(out_dtype) + + NK_SIZES_PER_TENSOR = [ (32, 64), (5120, 5120), @@ -274,3 +362,136 @@ def test_cpu_fp8_per_tensor_kernel_rejects_non_multiple_of_32_n(n: int): ) supported, _ = CPUFp8PerTensorScaledMMLinearKernel.can_implement(config) assert not supported + + +def reference_dynamic_fp8_linear( + x: torch.Tensor, # BF16 [M, K] + weight_fp8: torch.Tensor, # FP8 [N, K] + weight_scale: torch.Tensor, # float32 [N, G] + bias: torch.Tensor | None, + out_dtype: torch.dtype, +) -> torch.Tensor: + """Dynamic per-token act x per-group weight: quantize x per-token, + dequant weight per-group, matmul.""" + N, K = weight_fp8.shape + G = weight_scale.shape[1] + group_size = K // G + x_q, x_scale = quantize_act_per_token(x) + x_dq = x_q.float() * x_scale + w_dq = ( + weight_fp8.float().view(N, G, group_size) * weight_scale.view(N, G, 1) + ).view(N, K) + out = torch.mm(x_dq, w_dq.t()) + if bias is not None: + out = out + bias.float() + return out.to(out_dtype) + + +# --------------------------------------------------------------------------- +# W8A8 tests: fp8_scaled_mm_with_quant +# --------------------------------------------------------------------------- +W8A8_M_SIZES = [1, 4, 16, 64] +W8A8_NK_SIZES = [(128, 256), (256, 512), (64, 128)] + + +@requires_cpu_fp8_w8a8 +@pytest.mark.parametrize("static_activation", [True, False]) +@pytest.mark.parametrize("M", W8A8_M_SIZES) +@pytest.mark.parametrize("N,K", W8A8_NK_SIZES) +@pytest.mark.parametrize("use_bias", [False, True]) +def test_cpu_fp8_w8a8_per_tensor( + M: int, N: int, K: int, use_bias: bool, static_activation: bool +): + """CPUFP8W8A8ScaledMMLinearKernel with per-tensor weight scale.""" + torch.manual_seed(42) + out_dtype = torch.bfloat16 + + x = torch.randn(M, K, dtype=torch.bfloat16) + weight = torch.randn(N, K, dtype=torch.bfloat16) + bias = torch.randn(N, dtype=torch.float32) if use_bias else None + + w_fp8, w_scale = quantize_weight_per_tensor(weight) + packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale) + + if static_activation: + act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1) + kernel_out = ops.fp8_scaled_mm_with_quant( + x, act_scale, False, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_static_fp8_linear( + x, w_fp8, w_scale, act_scale, bias, out_dtype + ) + atol = 0.05 + else: + kernel_out = ops.fp8_scaled_mm_with_quant( + x, None, True, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype) + atol = 0.05 + + assert kernel_out.dtype == out_dtype + torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol) + + +@requires_cpu_fp8_w8a8 +@pytest.mark.parametrize("static_activation", [True, False]) +@pytest.mark.parametrize("M", W8A8_M_SIZES) +@pytest.mark.parametrize("N,K", W8A8_NK_SIZES) +@pytest.mark.parametrize("use_bias", [False, True]) +def test_cpu_fp8_w8a8_per_channel( + M: int, N: int, K: int, use_bias: bool, static_activation: bool +): + """CPUFP8W8A8ScaledMMLinearKernel with per-channel weight scale.""" + torch.manual_seed(42) + out_dtype = torch.bfloat16 + + x = torch.randn(M, K, dtype=torch.bfloat16) + weight = torch.randn(N, K, dtype=torch.bfloat16) + bias = torch.randn(N, dtype=torch.float32) if use_bias else None + + w_fp8, w_scale = quantize_weight_per_channel(weight) + packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale) + + if static_activation: + act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1) + kernel_out = ops.fp8_scaled_mm_with_quant( + x, act_scale, False, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_static_fp8_linear( + x, w_fp8, w_scale, act_scale, bias, out_dtype + ) + atol = 0.05 + else: + kernel_out = ops.fp8_scaled_mm_with_quant( + x, None, True, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype) + atol = 0.05 + + assert kernel_out.dtype == out_dtype + torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol) + + +@requires_cpu_fp8_w8a8 +@pytest.mark.parametrize("M", W8A8_M_SIZES) +@pytest.mark.parametrize("N,K", W8A8_NK_SIZES) +@pytest.mark.parametrize("use_bias", [False, True]) +def test_cpu_fp8_w8a8_per_group(M: int, N: int, K: int, use_bias: bool): + """CPUFp8W8A8BlockScaledMMKernel: per-group (128-block) weight scale + + dynamic per-token activation scale.""" + torch.manual_seed(42) + out_dtype = torch.bfloat16 + + x = torch.randn(M, K, dtype=torch.bfloat16) + weight = torch.randn(N, K, dtype=torch.bfloat16) + bias = torch.randn(N, dtype=torch.float32) if use_bias else None + + w_fp8, w_scale = quantize_weight_per_group(weight) + packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale) + kernel_out = ops.fp8_scaled_mm_with_quant( + x, None, True, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype) + + assert kernel_out.dtype == out_dtype + torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=0.05) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 96f47e2069..2a1654231f 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from enum import IntEnum +from functools import cache from typing import TYPE_CHECKING, Literal import torch @@ -3384,6 +3385,7 @@ class CPUQuantMethod(IntEnum): FP8_W8A16 = 2 INT4_W4A8 = 3 MXFP4 = 4 + FP8_W8A8 = 5 if hasattr(torch.ops._C, "dynamic_4bit_int_moe"): @@ -3422,6 +3424,7 @@ def fused_experts_cpu( alpha: float | None = None, limit: float | None = None, is_vnni: bool = True, + a1_scale: torch.Tensor | None = None, ) -> None: torch.ops._C.fused_experts_cpu( out, @@ -3435,6 +3438,7 @@ def fused_experts_cpu( w2_scale, w1_zero, w2_zero, + a1_scale, block_size, w1_bias, w2_bias, @@ -3564,6 +3568,46 @@ def fp8_scaled_mm_cpu( ) +# FP8 W8A8 CPU kernels +@cache +def cpu_has_amx_fp8() -> bool: + """Whether this CPU has native AMX-FP8 MMA support.""" + if not hasattr(torch.ops._C, "cpu_has_amx_fp8"): + return False + return bool(torch.ops._C.cpu_has_amx_fp8()) + + +if hasattr(torch.ops._C, "fp8_scaled_mm_with_quant"): + + @register_fake("_C::fp8_scaled_mm_with_quant") + def fp8_scaled_mm_with_quant_fake( + act: torch.Tensor, + act_scales: torch.Tensor | None, + channelwise: bool, + weight: torch.Tensor, + weight_scales: torch.Tensor, + bias: torch.Tensor | None, + output_dtype: torch.dtype, + ) -> torch.Tensor: + M = act.reshape(-1, act.size(-1)).size(0) + N = weight.size(0) * weight.size(-1) + return torch.empty((M, N), dtype=output_dtype, device=act.device) + + +def fp8_scaled_mm_with_quant( + act: torch.Tensor, + act_scales: torch.Tensor | None, + channelwise: bool, + weight: torch.Tensor, + weight_scales: torch.Tensor, + bias: torch.Tensor | None, + output_dtype: torch.dtype, +) -> torch.Tensor: + return torch.ops._C.fp8_scaled_mm_with_quant( + act, act_scales, channelwise, weight, weight_scales, bias, output_dtype + ) + + def chunk_gated_delta_rule_cpu( query: torch.Tensor, key: torch.Tensor, diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 8c57059e57..fb7a170b82 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -186,6 +186,8 @@ from vllm.model_executor.kernels.linear.scaled_mm.b12x import ( from vllm.model_executor.kernels.linear.scaled_mm.cpu import ( CPUFp8BlockScaledMMKernel, CPUFp8PerTensorScaledMMLinearKernel, + CPUFp8W8A8BlockScaledMMKernel, + CPUFP8W8A8ScaledMMLinearKernel, CPUInt8ScaledMMLinearKernel, ) from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( @@ -436,6 +438,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = ChannelWiseTorchFP8ScaledMMLinearKernel, ], PlatformEnum.CPU: [ + CPUFP8W8A8ScaledMMLinearKernel, CPUFp8PerTensorScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel, ChannelWiseTorchFP8ScaledMMLinearKernel, @@ -468,6 +471,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[ TritonFp8BlockScaledMMKernel, ], PlatformEnum.CPU: [ + CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below CPUFp8BlockScaledMMKernel, ], PlatformEnum.XPU: [ diff --git a/vllm/model_executor/kernels/linear/scaled_mm/__init__.py b/vllm/model_executor/kernels/linear/scaled_mm/__init__.py index f4a062b7f2..75db4bcaa9 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/__init__.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/__init__.py @@ -10,6 +10,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel impo from vllm.model_executor.kernels.linear.scaled_mm.cpu import ( CPUFp8BlockScaledMMKernel, CPUFp8PerTensorScaledMMLinearKernel, + CPUFP8W8A8ScaledMMLinearKernel, CPUInt8ScaledMMLinearKernel, ) from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( @@ -69,5 +70,6 @@ __all__ = [ "Fp8BlockScaledMMLinearKernel", "CPUFp8BlockScaledMMKernel", "CPUFp8PerTensorScaledMMLinearKernel", + "CPUFP8W8A8ScaledMMLinearKernel", "XPUFp8BlockScaledMMKernel", ] diff --git a/vllm/model_executor/kernels/linear/scaled_mm/cpu.py b/vllm/model_executor/kernels/linear/scaled_mm/cpu.py index e72dffcbbc..1d2ede1853 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/cpu.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/cpu.py @@ -270,12 +270,17 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): # Skip the base class process (FP8 padding / fnuz normalization) # which is GPU-oriented. Instead, VNNI-prepack weights for AMX. params = self._get_layer_params(layer) - packed_weight = torch.ops._C.convert_weight_packed(params.weight) - replace_parameter( - layer, - params.WEIGHT, - torch.nn.Parameter(packed_weight, requires_grad=False), - ) + + # MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): MLA reads this + # weight to derive W_UK/W_UV, so keep the checkpoint [N, K] layout and + # let the kernel pack on the fly instead. + if not getattr(layer, "_cpu_skip_gemm_dispatch", False): + packed_weight = torch.ops._C.convert_weight_packed(params.weight) + replace_parameter( + layer, + params.WEIGHT, + torch.nn.Parameter(packed_weight, requires_grad=False), + ) # Re-wrap scale as a plain Parameter so the kernel can read it # without weight-loader metadata interfering. @@ -320,7 +325,7 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): list(self.weight_group_shape), bias, x.dtype, - True, # is_vnni (weight already prepacked) + not getattr(layer, "_cpu_skip_gemm_dispatch", False), # is_vnni ) return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out @@ -430,3 +435,294 @@ class CPUFp8PerTensorScaledMMLinearKernel(FP8ScaledMMLinearKernel): raise NotImplementedError( "CPUFp8PerTensorScaledMMLinearKernel overrides apply_weights directly." ) + + +class CPUFp8W8A8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): + """FP8 W8A8 block-quantized GEMM with dynamic per-token activation quant on CPU.""" + + # Activation quant is done inside fp8_scaled_mm_with_quant; + # the base class must not apply a second quant step. + apply_input_quant = False + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cpu(): + return False, "requires CPU platform." + if not torch.cpu._is_amx_tile_supported(): + return False, "requires AMX tile support (Sapphire Rapids or newer)." + if not ops.cpu_has_amx_fp8(): + return False, ( + "requires native AMX-FP8 hardware; falls back " + "to CPUFp8BlockScaledMMKernel (W8A16) otherwise." + ) + return True, None + + @classmethod + def can_implement( + cls, config: FP8ScaledMMLinearLayerConfig + ) -> tuple[bool, str | None]: + # Same block-shape requirements as the W8A16 sibling kernel. + weight_gs = config.weight_quant_key.scale.group_shape + if weight_gs.col <= 0 or weight_gs.col != 128: + return False, ( + "CPU FP8 W8A8 block kernel requires K-dimension block size of 128, " + f"got {weight_gs.col}." + ) + if weight_gs.row <= 0 or weight_gs.row % 32 != 0: + return False, ( + "CPU FP8 W8A8 block kernel requires N-dimension block size to be " + f"a positive multiple of 32, got {weight_gs.row}." + ) + if config.out_dtype not in (torch.bfloat16, torch.float32): + return False, "Only bfloat16/float32 output dtype supported." + n = config.weight_shape[0] + if n % 32 != 0: + # float8_linear_prepack_cpu tiles N in chunks of 32 and cannot + # handle a remainder tile smaller than that. + return False, f"requires weight output dim (N={n}) to be a multiple of 32." + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Prepack block-quantized FP8 weight for W8A8 AMX BRGEMM. + + For block-quant FP8 models weight is stored as [N, K] (NOT transposed, + unlike non-block FP8 where vLLM stores [K, N]). + + float8_linear_prepack_cpu expects: + weight: [N, K] + scales: [N, G] where G = K / block_k + + weight_scale_inv from the checkpoint has shape [ceil(N/block_n), G], + so we expand rows from ceil(N/block_n) to N via repeat_interleave. + """ + params = self._get_layer_params(layer) + weight = params.weight # [N, K] + + # MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): MLA reads this + # weight to derive W_UK/W_UV, so keep the checkpoint [N, K] layout and + # its block scales; apply_weights falls back to the W8A16 op. + if getattr(layer, "_cpu_skip_gemm_dispatch", False): + return + + weight_scale = ( + params.weight_scale_inv + if params.weight_scale_inv is not None + else params.weight_scale + ) # [ceil(N/block_n), ceil(K/block_k)] + assert weight_scale is not None, "FP8 W8A8 block kernel requires weight scale." + + N = weight.shape[0] + block_n = self.weight_group_shape.row # e.g. 128 + + # Expand scale rows: [ceil(N/block_n), G] → [N, G] + if weight_scale.size(0) != N: + weight_scale_expanded = weight_scale.repeat_interleave(block_n, dim=0)[ + :N, : + ].contiguous() + else: + weight_scale_expanded = weight_scale.contiguous() + + packed_weight, packed_scales = torch.ops._C.float8_linear_prepack_cpu( + weight, weight_scale_expanded.to(torch.float32) + ) + + scale_attr = ( + params.WEIGHT_SCALE_INV + if params.weight_scale_inv is not None + else params.WEIGHT_SCALE + ) + replace_parameter( + layer, + params.WEIGHT, + torch.nn.Parameter(packed_weight, requires_grad=False), + ) + replace_parameter( + layer, + scale_attr, + torch.nn.Parameter(packed_scales, requires_grad=False), + ) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + params = self._get_layer_params(layer) + weight_scale = ( + params.weight_scale_inv + if params.weight_scale_inv is not None + else params.weight_scale + ) + + x_2d = x.reshape(-1, x.shape[-1]) if x.dim() > 2 else x + if getattr(layer, "_cpu_skip_gemm_dispatch", False): + # Unpacked weight: fp8_scaled_mm_with_quant has no is_vnni option, + # so fall back to the W8A16 op (BF16 activation, block-scaled FP8 + # weight) for this layer. + out = torch.ops._C.fp8_scaled_mm_cpu( + x_2d, + params.weight, + weight_scale, + list(self.weight_group_shape), + bias, + x.dtype, + False, # is_vnni + ) + return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out + + # act_scales=None + channelwise=True → dynamic per-token activation quant + out = ops.fp8_scaled_mm_with_quant( + x_2d, + None, + True, + params.weight, + weight_scale, + bias, + x.dtype, + ) + return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out + + def apply_block_scaled_mm( + self, + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + ) -> torch.Tensor: + raise NotImplementedError( + "CPUFp8W8A8BlockScaledMMKernel overrides apply_weights directly." + ) + + +class CPUFP8W8A8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): + """FP8 W8A8 GEMM with dynamic per-token activation quantization on CPU.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cpu(): + return False, "requires CPU platform." + if not torch.cpu._is_amx_tile_supported(): + return False, "requires AMX tile support (Sapphire Rapids or newer)." + if not ops.cpu_has_amx_fp8(): + return False, ( + "requires native AMX-FP8 hardware (Diamond Rapids); falls back " + "to the Torch FP8 kernels otherwise." + ) + return True, None + + @classmethod + def can_implement( + cls, config: FP8ScaledMMLinearLayerConfig + ) -> tuple[bool, str | None]: + if config.out_dtype not in (torch.bfloat16, torch.float16, torch.float32): + return False, "Only bfloat16/float16/float32 output dtype supported." + # Reject block-quantized weights — those go to CPUFp8BlockScaledMMKernel + weight_gs = config.weight_quant_key.scale.group_shape + if weight_gs.col > 0 and weight_gs.row > 0: + return False, "Block-quantized weights not handled by FP8 W8A8 kernel." + n = config.weight_shape[0] + if n % 32 != 0: + # float8_linear_prepack_cpu tiles N in chunks of 32 and cannot + # handle a remainder tile smaller than that. + return False, f"requires weight output dim (N={n}) to be a multiple of 32." + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Prepack weights with float8_linear_prepack_cpu (VNNI + block layout).""" + w_name, ws_name, xs_name, _ = self.layer_param_names + weight = getattr(layer, w_name) + weight_scale = getattr(layer, ws_name) + + # vLLM stores linear weight as [K, N], while the CPU prepack op expects [N, K]. + N = weight.shape[-1] + + # Ensure scale is 2D [N, G] + ws = weight_scale + if ws is None: + raise RuntimeError("FP8 W8A8 kernel requires weight scale.") + ws = ws.to(torch.float32) + + # Fused modules may carry per-shard scales (e.g. QKV/MLP shards). + # Expand those to per-output-channel scales before prepack. + if ( + len(getattr(layer, "logical_widths", [])) > 1 + and ws.dim() <= 1 + and ws.numel() == len(layer.logical_widths) + ): + ws = convert_to_channelwise(ws.reshape(-1), layer.logical_widths) + + if ws.dim() == 0: + ws = ws.view(1, 1).expand(N, 1) + elif ws.dim() == 1: + if ws.numel() == 1: + ws = ws.view(1, 1).expand(N, 1) + elif ws.numel() == N: + ws = ws.unsqueeze(1) # [N] -> [N, 1] + else: + raise RuntimeError( + "FP8 W8A8 kernel got unsupported 1D weight scale shape: " + f"{tuple(ws.shape)} for N={N}." + ) + + # Align scale rows to output channels N. + if ws.size(0) != N: + if ws.size(0) == 1: + ws = ws.expand(N, ws.size(1)).contiguous() + else: + raise RuntimeError( + "FP8 W8A8 kernel got mismatched weight scale rows: " + f"ws_rows={ws.size(0)}, expected N={N}." + ) + + w_2d = weight.t().contiguous() # [N, K] + packed_weight, packed_scales = torch.ops._C.float8_linear_prepack_cpu(w_2d, ws) + + replace_parameter( + layer, + w_name, + torch.nn.Parameter(packed_weight, requires_grad=False), + ) + replace_parameter( + layer, + ws_name, + torch.nn.Parameter(packed_scales, requires_grad=False), + ) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + w_name, ws_name, xs_name, _ = self.layer_param_names + weight = getattr(layer, w_name) + weight_scale = getattr(layer, ws_name) + # For static quantization, use pre-computed activation scale + act_scales = getattr(layer, xs_name, None) if xs_name else None + + x_2d = x.reshape(-1, x.shape[-1]) if x.dim() > 2 else x + # channelwise=True: per-token (per-row) quantization when act_scales is None + # channelwise=False: per-tensor quantization when act_scales is provided + channelwise = act_scales is None + out = ops.fp8_scaled_mm_with_quant( + x_2d, + act_scales, + channelwise, + weight, + weight_scale, + bias, + x.dtype, + ) + return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out + + def apply_scaled_mm(self, A, B, out_dtype, As, Bs, bias=None, **kwargs): + raise NotImplementedError( + "CPUFP8W8A8ScaledMMLinearKernel uses apply_weights directly." + ) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 26547c7353..3104ac25c6 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1055,7 +1055,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): mqa_q_nope, self.impl._w_uk_packed, # type: ignore[attr-defined] True, - None, + self.impl._w_scale, # type: ignore[attr-defined] ) mqa_ql_nope = mqa_ql_nope.transpose(0, 1) else: @@ -1401,7 +1401,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): x, self.impl._w_uv_packed, # type: ignore[attr-defined] True, - None, + self.impl._w_scale, # type: ignore[attr-defined] ) else: # Multiply + Transpose (N, B, L) x (N, L, V)->(N, B, V)->(B, N, V) diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 858b351864..3e7376fe90 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -15,6 +15,7 @@ from vllm._custom_ops import ( convert_weight_packed_scale_zp, cpu_fused_moe, cpu_fused_moe_int8, + cpu_has_amx_fp8, cpu_prepack_moe_weight, cpu_prepack_moe_weight_int8, fused_experts_cpu, @@ -467,6 +468,7 @@ class CPUExpertsFp8(mk.FusedMoEExpertsModular): RoutingMethodType.Default, RoutingMethodType.Renormalize, RoutingMethodType.RenormalizeNaive, + RoutingMethodType.DeepSeekV3, RoutingMethodType.DeepseekV4, RoutingMethodType.DeepSeekV3, ] @@ -1612,3 +1614,218 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular): self._w1_scale_bf16, self._w2_scale_bf16, ) + + +# =========================================================================== +# FP8 W8A8 MoE +# =========================================================================== + + +def prepare_fp8_w8a8_moe_layer_for_cpu( + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Prepack FP8 W8A8 MoE weights for CPU kernel.""" + E = w13.size(0) + + packed_w13_list = [] + packed_w13_scale_list = [] + + for i in range(E): + w1_exp = w13[i] # [2N, K] FP8 + ws_exp = w13_scale[i] # [2N, K/128] or [2N_blocks, K/128] + + # Align scale rows to weight rows if necessary (for block-128 checkpoints + # where scale may have fewer rows: [2N/128, K/128] → [2N, K/128]) + if ws_exp.size(0) < w1_exp.size(0): + repeat_factor = w1_exp.size(0) // ws_exp.size(0) + ws_exp = torch.repeat_interleave(ws_exp, repeat_factor, dim=0) + ws_exp = ws_exp[: w1_exp.size(0), :].contiguous() + + pw, ps = torch.ops._C.float8_linear_prepack_cpu(w1_exp, ws_exp) + packed_w13_list.append(pw) + packed_w13_scale_list.append(ps) + + packed_w13 = torch.stack(packed_w13_list) + packed_w13_scale = torch.stack(packed_w13_scale_list) + + # w2 uses the W8A16 path (BF16 activation × FP8 weight), VNNI packed + packed_w2 = torch.ops._C.convert_weight_packed(w2) + return packed_w13, packed_w13_scale, packed_w2, w2_scale + + +class CPUExpertsFp8W8A8(mk.FusedMoEExpertsModular): + """CPU FP8 W8A8 block-quantized modular MoE experts.""" + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + + @property + def expects_unquantized_inputs(self) -> bool: + # Receives BF16 hidden states; quantization to FP8 is done inside apply() + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + # Requires native AMX-FP8 hardware. + return current_platform.is_cpu() and cpu_has_amx_fp8() + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation == MoEActivation.SILU + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return True + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + # Block-quantized FP8. + return (weight_key, activation_key) == ( + kFp8Static128BlockSym, + kFp8Dynamic128Sym, + ) + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + RoutingMethodType.DeepSeekV3, + RoutingMethodType.DeepseekV4, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def moe_problem_size( + self, + a1: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_ids: torch.Tensor, + ) -> tuple[int, int, int, int, int]: + # float8_linear_prepack_cpu blocks w1 into [E, Nc, Kc, block_k, block_n], + # not the (E, N, K) layout the base implementation assumes -- N isn't + # recoverable from that shape, so read it from moe_config instead. + E = w1.shape[0] + K = a1.size(-1) + N = ( + self.moe_config.intermediate_size_per_partition + * self.moe_config.w13_num_shards + ) + M = a1.size(0) if a1.dim() == 2 else a1.size(1) + topk = topk_ids.size(1) + return E, M, N, K, topk + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + # fused_experts_cpu manages its own scratch space. + return (0,), (0,), (M, K) + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ) -> None: + if apply_router_weight_on_input: + # fused_experts_cpu always applies topk_weights internally on + # combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would + # also pre-apply it to hidden_states, double-weighting the + # output. Not needed by any CPU FP8 model today. + raise NotImplementedError( + "CPUExpertsFp8W8A8 does not support apply_router_weight_on_input=True." + ) + + block_shape = ( + list(self.quant_config.block_shape) + if self.quant_config.block_shape + else ( + [ + self.quant_config._w1.shape.row, + self.quant_config._w1.shape.col, + ] + if self.quant_config._w1.shape is not None + else None + ) + ) + + # Quantize hidden_states (BF16) → FP8 with per-token scales + hidden_states_2d = hidden_states.reshape(-1, hidden_states.shape[-1]) + x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec( + hidden_states_2d, True, None + ) + + fused_experts_cpu( + output, + x_fp8, + w1, + w2, + topk_weights, + topk_ids, + CPUQuantMethod.FP8_W8A8, + self.w1_scale, # w1_scale + self.w2_scale, # w2_scale + None, # w1_zero + None, # w2_zero + block_shape, # block_size + None, # w1_bias + None, # w2_bias + None, # alpha + None, # limit + True, # is_vnni + x_scales, # a1_scale: per-token FP8 activation scales + ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 6a6ce34d16..abf0896204 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -57,7 +57,9 @@ class Fp8MoeBackend(Enum): VLLM_CUTLASS = "VLLM_CUTLASS" BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS" XPU = "XPU" + # CPU FP8 W8A16 (BF16 activations); CPU_W8A8 needs native AMX-FP8. CPU = "CPU" + CPU_W8A8 = "CPU_W8A8" HPC = "HPC" # Dequantize-to-BF16 emulation for MXFP8 on devices without a native # MXFP8 MoE kernel (e.g. ROCm). Weights pass through unchanged here. @@ -92,6 +94,7 @@ def _get_priority_backends( Fp8MoeBackend.BATCHED_VLLM_CUTLASS, Fp8MoeBackend.BATCHED_TRITON, Fp8MoeBackend.XPU, + Fp8MoeBackend.CPU_W8A8, Fp8MoeBackend.CPU, Fp8MoeBackend.HPC, ] @@ -129,8 +132,10 @@ def _get_priority_backends( _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.XPU) if current_platform.is_cpu(): - # CPU platform uses FP8 W8A16 fused MoE kernel. + # W8A8 first: it falls through to the W8A16 backend whenever the + # hardware (AMX-FP8) or the config isn't supported. _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU) + _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU_W8A8) return _AVAILABLE_BACKENDS @@ -238,6 +243,13 @@ def backend_to_kernel_cls( return [CPUExpertsFp8] + elif backend == Fp8MoeBackend.CPU_W8A8: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + CPUExpertsFp8W8A8, + ) + + return [CPUExpertsFp8W8A8] + elif backend == Fp8MoeBackend.HPC: from vllm.model_executor.layers.fused_moe.hpc_moe import ( HPCExperts, @@ -652,6 +664,14 @@ def convert_to_fp8_moe_kernel_format( ) w13, w2 = prepare_fp8_moe_layer_for_cpu(w13, w2) + elif fp8_backend == Fp8MoeBackend.CPU_W8A8: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + prepare_fp8_w8a8_moe_layer_for_cpu, + ) + + w13, w13_scale, w2, w2_scale = prepare_fp8_w8a8_moe_layer_for_cpu( + w13, w2, w13_scale, w2_scale + ) else: if fp8_backend not in [ Fp8MoeBackend.TRITON, @@ -697,7 +717,15 @@ def make_fp8_moe_quant_config( In a future PR, we will have this function should be a method of the modular kernel itself. """ - # MARLIN and CPU are mixed precision W8A16 config. + if fp8_backend == Fp8MoeBackend.CPU_W8A8: + return fp8_w8a8_moe_quant_config( + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + block_shape=block_shape, + ) + + # MARLIN and CPU (W8A16) are mixed precision W8A16 configs. if fp8_backend == Fp8MoeBackend.MARLIN or fp8_backend == Fp8MoeBackend.CPU: return fp8_w8a16_moe_quant_config( w1_scale=w1_scale, diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 2a8ef0b69f..ed018c81bf 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -737,11 +737,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase): ) # Reject backends whose make_fp8_moe_quant_config branch silently # drops per_act_token_quant / per_out_ch_quant or collapses scales: - # MARLIN / CPU route through fp8_w8a16_moe_quant_config; FLASHINFER_* - # fold scales into a per-tensor alpha (oracle/fp8.py). + # MARLIN / CPU route through fp8_w8a16_moe_quant_config, CPU_W8A8 + # ignores both flags, and FLASHINFER_* fold scales into a per-tensor + # alpha (oracle/fp8.py). if self.fp8_backend in ( Fp8MoeBackend.MARLIN, Fp8MoeBackend.CPU, + Fp8MoeBackend.CPU_W8A8, Fp8MoeBackend.FLASHINFER_CUTLASS, Fp8MoeBackend.FLASHINFER_TRTLLM, ): diff --git a/vllm/v1/attention/backends/mla/amx_mla.py b/vllm/v1/attention/backends/mla/amx_mla.py index 5f9ac7f17a..38c8b68af9 100644 --- a/vllm/v1/attention/backends/mla/amx_mla.py +++ b/vllm/v1/attention/backends/mla/amx_mla.py @@ -61,6 +61,44 @@ _MIN_WORK_PER_SPLIT = 512 _SPLIT_OCCUPANCY_MULTIPLIER = 2 +def _block_quant_to_tensor_quant( + weight: torch.Tensor, # FP8 [N, K] block-quantized weight + weight_scale: torch.Tensor, # [ceil(N/bn), ceil(K/bk)] block scales + block_size: list[int], # [block_n, block_k] +) -> tuple[torch.Tensor, torch.Tensor]: + """Re-quantize FP8 block-quant weight to FP8 per-tensor quant. + + Returns (fp8_weight, dequant_scale) where fp8_weight * dequant_scale + approximates the original float weight. dequant_scale is a 0-dim + scalar tensor as expected by bmm_cpu's ``scale`` argument. + + NOTE: block->tensor re-quantization may slightly affect accuracy + (equivalent to SGLang's block_quant_to_tensor_quant for CPU). + """ + block_n, block_k = block_size[0], block_size[1] + n, k = weight.shape + + # Expand block scales [ceil(N/bn), ceil(K/bk)] -> [N, K] + scale = weight_scale.to(torch.float32) + if scale.shape[0] < n: + scale = scale.repeat_interleave(block_n, dim=0)[:n, :] + if scale.shape[1] < k: + scale = scale.repeat_interleave(block_k, dim=1)[:, :k] + + # Dequantize to float32 + x_f32 = weight.to(torch.float32) * scale + + # Re-quantize per-tensor back to FP8 + fp8_max = torch.finfo(weight.dtype).max # 448.0 for e4m3fn + amax = x_f32.abs().amax().clamp(min=1e-12) + quant_scale = fp8_max / amax + x_fp8 = (x_f32 * quant_scale).clamp(-fp8_max, fp8_max).to(weight.dtype) + + # dequant_scale = 1 / quant_scale (0-dim tensor for bmm_cpu) + dequant_scale = (amax / fp8_max).to(torch.float32) + return x_fp8, dequant_scale + + def _compute_num_kv_splits(max_seq_len: int, num_threads: int) -> int: """Mirrors TritonMLAImpl's _compute_num_kv_splits, using the CPU thread count in place of SM count.""" @@ -208,6 +246,9 @@ class AMXMLAImpl(MLACommonImpl[MLACommonMetadata]): ) self._w_uk_packed: torch.Tensor | None = None self._w_uv_packed: torch.Tensor | None = None + self._w_scale: torch.Tensor | None = ( + None # per-tensor dequant scale (FP8 W8A16) + ) def process_weights_after_loading(self, act_dtype: torch.dtype) -> None: # forward_mha (unlike forward_mqa) receives no `layer` argument, so it @@ -216,12 +257,40 @@ class AMXMLAImpl(MLACommonImpl[MLACommonMetadata]): # impl derives its own copies from self.kv_b_proj (already available # via the constructor) and VNNI-packs them once, ahead of time, for # use with bmm_cpu in forward_mha. - kv_b_proj_weight = get_and_maybe_dequant_weights( - self.kv_b_proj, out_dtype=act_dtype - ).T - kv_b_proj_weight = kv_b_proj_weight.view( - self.kv_lora_rank, self.num_heads, self.qk_nope_head_dim + self.v_head_dim - ) + # + # kv_b_proj carries `_cpu_skip_gemm_dispatch`, so the CPU FP8 kernels + # leave its weight in the checkpoint [N, K] layout. For block-quant + # FP8 checkpoints, re-quantize per-tensor and keep the BMM in FP8 + # (W8A16); everything else goes through the BF16 path. + weight = getattr(self.kv_b_proj, "weight", None) + block_size = getattr(self.kv_b_proj, "weight_block_size", None) + if ( + weight is not None + and weight.dtype == torch.float8_e4m3fn + and block_size is not None + ): + block_scale = getattr(self.kv_b_proj, "weight_scale_inv", None) + if block_scale is None: + block_scale = self.kv_b_proj.weight_scale + fp8_weight, self._w_scale = _block_quant_to_tensor_quant( + weight, block_scale, block_size + ) + # [N, K] -> [K, N] = [kv_lora_rank, heads * (nope + v)] + kv_b_proj_weight = fp8_weight.T.contiguous().view( + self.kv_lora_rank, + self.num_heads, + self.qk_nope_head_dim + self.v_head_dim, + ) + else: + kv_b_proj_weight = get_and_maybe_dequant_weights( + self.kv_b_proj, out_dtype=act_dtype + ).T + kv_b_proj_weight = kv_b_proj_weight.view( + self.kv_lora_rank, + self.num_heads, + self.qk_nope_head_dim + self.v_head_dim, + ) + self._w_scale = None w_uk, w_uv = kv_b_proj_weight.split( [self.qk_nope_head_dim, self.v_head_dim], dim=-1 ) @@ -348,7 +417,9 @@ class AMXMLAImpl(MLACommonImpl[MLACommonMetadata]): ql_nope = torch.empty( num_tokens, num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device ) - ops.bmm_cpu(ql_nope.transpose(0, 1), q_nope_t, self._w_uk_packed, True, None) + ops.bmm_cpu( + ql_nope.transpose(0, 1), q_nope_t, self._w_uk_packed, True, self._w_scale + ) mqa_q = torch.cat([ql_nope, q_pe], dim=-1) kv_cache_flat = kv_c_and_k_pe_cache.view(-1, 1, self.head_size) @@ -402,4 +473,4 @@ class AMXMLAImpl(MLACommonImpl[MLACommonMetadata]): output_view = output.view(num_tokens, num_heads, self.v_head_dim).transpose( 0, 1 ) - ops.bmm_cpu(output_view, attn_out_t, self._w_uv_packed, True, None) + ops.bmm_cpu(output_view, attn_out_t, self._w_uv_packed, True, self._w_scale) From 5bb596201a19ba73e08e05e062a7ad64e96ecda2 Mon Sep 17 00:00:00 2001 From: vllm-agent Date: Fri, 18 Sep 2026 03:10:38 -0700 Subject: [PATCH 088/143] [CI] Raise Elastic EP Scaling step timeout 30m -> 40m (#57287) Signed-off-by: kimi-no-na-wa Co-authored-by: kimi-no-na-wa --- .buildkite/test_areas/expert_parallelism.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index f74b1adf2e..485d4f3d31 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -43,7 +43,7 @@ steps: - label: ":nvidia: (H100) Elastic EP Scaling" key: elastic-ep-scaling-test - timeout_in_minutes: 30 + timeout_in_minutes: 40 device: h100 working_dir: "/vllm-workspace/tests" num_devices: 4 From 5397f967b9ee282354f9803af3e758c9270588e9 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Fri, 18 Sep 2026 03:11:07 -0700 Subject: [PATCH 089/143] [CI] Add residual timeout headroom after JIT rollback (#56649) Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: OpenAI Codex --- .buildkite/test_areas/engine.yaml | 2 +- .buildkite/test_areas/spec_decode.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 179954740a..f6e243a1b7 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -81,7 +81,7 @@ steps: - label: ":nvidia: (H200 MIG 18GB) E2E Core" device: h200_18gb key: e2e-core-1-gpu - timeout_in_minutes: 50 + timeout_in_minutes: 65 source_file_dependencies: - vllm/v1/ - tests/v1/e2e/general/ diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index fdb12efe11..f60d745df3 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -174,7 +174,7 @@ steps: - label: ":nvidia: (H200 MIG 18GB) Spec Decode Draft Model" key: spec-decode-draft-model - timeout_in_minutes: 45 + timeout_in_minutes: 60 device: h200_18gb source_file_dependencies: - vllm/v1/spec_decode/ From 4c6c1a40b3cb4f9018e2a6c79fdbe11621acbda1 Mon Sep 17 00:00:00 2001 From: Matt <156021403+mawong-amd@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:13:45 -0700 Subject: [PATCH 090/143] [ROCm][Bugfix] Fix intermittent ROCR host segfault (#57328) Signed-off-by: Matthew Wong --- docker/Dockerfile.rocm_base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index d842a7b07e..fa0a88fd20 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -15,7 +15,7 @@ ARG MORI_BRANCH="v1.1.0" ARG MORI_REPO="https://github.com/ROCm/mori.git" ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350" -ARG ROCM_RUNTIME_COMMIT="b539bf7eebfd99ad0a69668caa1f4037034d501f" +ARG ROCM_RUNTIME_COMMIT="af70edc91f1dc5d228fd4c076df99e65e49c4521" # Sccache configuration (only used in release pipeline) ARG USE_SCCACHE From f2b6152ec2448b4ea877153d6a613c381e7fd8bb Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Fri, 18 Sep 2026 03:40:32 -0700 Subject: [PATCH 091/143] [CI] Add flaky rerun markers / timeout skips to sibling tests lacking them (#57359) Signed-off-by: khluu Co-authored-by: Kimi --- .../pooling/test_token_classification.py | 2 ++ tests/multimodal/media/test_connector.py | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/models/language/pooling/test_token_classification.py b/tests/models/language/pooling/test_token_classification.py index d8d813f404..c205a3c42a 100644 --- a/tests/models/language/pooling/test_token_classification.py +++ b/tests/models/language/pooling/test_token_classification.py @@ -115,6 +115,7 @@ def test_modernbert_models( @pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"]) @pytest.mark.parametrize("dtype", ["float"]) +@pytest.mark.flaky(reruns=3) @torch.inference_mode def test_xlm_roberta_models( hf_runner, @@ -197,6 +198,7 @@ def test_openai_privacy_filter( @pytest.mark.parametrize("model", ["bd2lcco/Qwen3-0.6B-finetuned"]) @pytest.mark.parametrize("dtype", ["float"]) +@pytest.mark.flaky(reruns=3) @torch.inference_mode def test_auto_conversion( hf_runner, diff --git a/tests/multimodal/media/test_connector.py b/tests/multimodal/media/test_connector.py index b222d57d3c..5949c66b56 100644 --- a/tests/multimodal/media/test_connector.py +++ b/tests/multimodal/media/test_connector.py @@ -299,6 +299,7 @@ async def test_fetch_video_http(video_url: str, num_frames: int): assert metadata_sync == metadata_async +@pytest.mark.flaky(reruns=3, reruns_delay=5) @pytest.mark.asyncio @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS) @pytest.mark.parametrize("max_duration", [1, 60, 1800]) @@ -320,8 +321,11 @@ async def test_fetch_video_http_with_dynamic_loader( } ) - video_sync, metadata_sync = connector.fetch_video(video_url) - video_async, metadata_async = await connector.fetch_video_async(video_url) + try: + video_sync, metadata_sync = connector.fetch_video(video_url) + video_async, metadata_async = await connector.fetch_video_async(video_url) + except (TimeoutError, asyncio.TimeoutError) as e: + pytest.skip(f"Timeout fetching video (CI network flakiness): {e}") assert np.array_equal(video_sync, video_async) assert metadata_sync == metadata_async @@ -379,6 +383,7 @@ def test_placeholder_range_extract_embeds_range(offset, is_embed, expected): assert pr.extract_embeds_range() == expected +@pytest.mark.flaky(reruns=3, reruns_delay=5) @pytest.mark.asyncio @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS) @pytest.mark.parametrize("num_frames", [-1, 32, 1800]) @@ -394,8 +399,12 @@ async def test_allowed_media_domains(video_url: str, num_frames: int): ], ) - video_sync, metadata_sync = connector.fetch_video(video_url) - video_async, metadata_async = await connector.fetch_video_async(video_url) + try: + video_sync, metadata_sync = connector.fetch_video(video_url) + video_async, metadata_async = await connector.fetch_video_async(video_url) + except (TimeoutError, asyncio.TimeoutError) as e: + pytest.skip(f"Timeout fetching video (CI network flakiness): {e}") + assert np.array_equal(video_sync, video_async) assert metadata_sync == metadata_async From 04a3a00eb79118dda521577caaaba2ba6b65bb33 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Fri, 18 Sep 2026 03:40:54 -0700 Subject: [PATCH 092/143] [CI] Raise GSM8K startup max wait to 2400s for flaky 30B+ MoE eval configs (#57361) Signed-off-by: khluu --- tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml | 2 ++ .../Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml | 2 ++ .../Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml | 2 ++ .../Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml | 2 ++ .../Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml b/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml index bc5c11fa53..141bf68162 100644 --- a/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml +++ b/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml @@ -3,6 +3,8 @@ accuracy_threshold: 0.90 num_questions: 1319 num_fewshot: 5 max_concurrency: 100 +# 30B+ MoE with EP/PCP: download + kernel autotuning can exceed the 1200s default. +startup_max_wait_seconds: 2400 # Leave headroom for FlashInfer CUTLASS autotuning after KV-cache allocation. server_args: >- --compilation-config '{"cudagraph_mode":"NONE"}' diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml index 5f10684d2e..f81eb1c188 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml @@ -2,6 +2,8 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 +# 30B MoE + DeepGEMM/DeepEP JIT compile can exceed the 1200s default startup wait. +startup_max_wait_seconds: 2400 server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log" env: VLLM_USE_DEEP_GEMM: "1" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml index 37e6039e96..3987065f6f 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml @@ -2,6 +2,8 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block" accuracy_threshold: 0.85 num_questions: 1319 num_fewshot: 5 +# 30B MoE + DeepGEMM/DeepEP JIT compile can exceed the 1200s default startup wait. +startup_max_wait_seconds: 2400 server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log" env: VLLM_USE_DEEP_GEMM: "1" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml index 6624cea1ef..004e3e15e3 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml @@ -2,4 +2,6 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 +# 30B MoE + CuTeDSL/DeepEP JIT compile can exceed the 1200s default startup wait. +startup_max_wait_seconds: 2400 server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --moe-backend=flashinfer_cutedsl" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml index f2d4588e3a..55892f2196 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml @@ -2,4 +2,6 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 +# 30B MoE + CuTeDSL/DeepEP JIT compile can exceed the 1200s default startup wait. +startup_max_wait_seconds: 2400 server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --moe-backend=flashinfer_cutedsl" From cc09352e5c50ca6b7d2c6d79b29d6d3b18df3046 Mon Sep 17 00:00:00 2001 From: adenzhou1350 Date: Fri, 18 Sep 2026 18:44:39 +0800 Subject: [PATCH 093/143] [Bugfix][Multimodal] Preserve DeepSeek V4 image block spacing (#56882) Signed-off-by: Xucheng Zhou Signed-off-by: Isotr0py Co-authored-by: OpenAI Codex Co-authored-by: Isotr0py Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../fixtures/deepseek_v4/test_input_5.json | 39 +++++++++++++++++++ .../fixtures/deepseek_v4/test_output_5.txt | 9 +++++ tests/tokenizers_/test_deepseek_v4.py | 14 +++---- vllm/renderers/deepseek_v4.py | 4 +- vllm/tokenizers/deepseek_v4_encoding.py | 6 +-- 5 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json create mode 100644 tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt diff --git a/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json b/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json new file mode 100644 index 0000000000..2a2fa6a89e --- /dev/null +++ b/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json @@ -0,0 +1,39 @@ +{ + "thinking_mode": "thinking", + "reasoning_effort": "max", + "messages": [ + { + "role": "system", + "content": "You are a helpful vision assistant." + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "请按“第一张、第二张”的顺序回答:第一张图" + }, + { + "type": "image_url", + "image_url": { + "url": "examples/images/carrots.jpeg" + } + }, + { + "type": "text", + "text": "和第二张图" + }, + { + "type": "image_url", + "image_url": { + "url": "examples/images/corn.jpeg" + } + }, + { + "type": "text", + "text": "中分别是什么食材?它们通常食用的部位分别是什么?" + } + ] + } + ] +} diff --git a/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt b/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt new file mode 100644 index 0000000000..1203dc8db1 --- /dev/null +++ b/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt @@ -0,0 +1,9 @@ +<|begin▁of▁sentence|>You are a helpful vision assistant.<|User|>请按“第一张、第二张”的顺序回答:第一张图 + +<|deepseek_image|> + +和第二张图 + +<|deepseek_image|> + +中分别是什么食材?它们通常食用的部位分别是什么?<|Assistant|> \ No newline at end of file diff --git a/tests/tokenizers_/test_deepseek_v4.py b/tests/tokenizers_/test_deepseek_v4.py index f7bc5909ba..4403c2a877 100644 --- a/tests/tokenizers_/test_deepseek_v4.py +++ b/tests/tokenizers_/test_deepseek_v4.py @@ -54,13 +54,9 @@ def _load_reference_case(case_id: int): def _render_reference_case(case_id: int, **kwargs): messages, tools = _load_reference_case(case_id) - conversation, _, _ = parse_chat_messages( - messages, - _model_config(), - content_format="string", - ) + # Preserve content blocks for encoding without fetching fixture media. return _tokenizer().apply_chat_template( - conversation=conversation, + conversation=messages, messages=messages, tools=tools, tokenize=False, @@ -397,6 +393,7 @@ def test_deepseek_v4_maps_xhigh_to_high_reasoning_effort(): (2, {"thinking": True, "reasoning_effort": "low"}), (3, {"thinking": True, "reasoning_effort": "low"}), (4, {"thinking": False}), + (5, {"thinking": False}), ], ) def test_deepseek_v4_matches_reference_golden_fixtures(case_id, kwargs): @@ -447,7 +444,10 @@ def test_deepseek_v4_image_blocks_become_placeholders(): thinking=False, ) - assert "<|User|>first:<|deepseek_image|>second:<|deepseek_image|>" in prompt + assert ( + "<|User|>first:\n\n<|deepseek_image|>\n\n" + "second:\n\n<|deepseek_image|>" in prompt + ) def test_deepseek_v4_image_sentinel_ids_match_tokenizer(): diff --git a/vllm/renderers/deepseek_v4.py b/vllm/renderers/deepseek_v4.py index e93069209a..ede2e9530c 100644 --- a/vllm/renderers/deepseek_v4.py +++ b/vllm/renderers/deepseek_v4.py @@ -40,7 +40,7 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): conversation, mm_data, mm_uuids = parse_chat_messages( messages, self.model_config, - content_format="string", + content_format="openai", media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, ) @@ -67,7 +67,7 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): conversation, mm_data, mm_uuids = await parse_chat_messages_async( messages, self.model_config, - content_format="string", + content_format="openai", media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, ) diff --git a/vllm/tokenizers/deepseek_v4_encoding.py b/vllm/tokenizers/deepseek_v4_encoding.py index fb5bc44e85..37dc559366 100644 --- a/vllm/tokenizers/deepseek_v4_encoding.py +++ b/vllm/tokenizers/deepseek_v4_encoding.py @@ -224,7 +224,7 @@ def flatten_content_blocks(content: Any) -> Any: raise ValueError( f"Unsupported content block type: {block.get('type')!r}" ) - return "".join(parts) + return "\n\n".join(parts) def render_message( @@ -267,7 +267,7 @@ def render_message( ) role = msg.get("role") - content = msg.get("content") + content = flatten_content_blocks(msg.get("content")) tools = msg.get("tools") response_format = msg.get("response_format") tool_calls = msg.get("tool_calls") @@ -337,7 +337,7 @@ def render_message( parts.append(f"[Unsupported {block_type}]") prompt += "\n\n".join(parts) else: - prompt += flatten_content_blocks(content) or "" + prompt += content or "" elif role == "latest_reminder": prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(content=content) From c58532c86b8ab821fc1d819636480b7588bce7b5 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 18 Sep 2026 19:44:05 +0800 Subject: [PATCH 094/143] [Frontend] Upgrade XGrammar to 0.2.7 and Rust structural tags to 0.3.0 (#57272) Co-authored-by: Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao --- requirements/common.txt | 2 +- requirements/test/cpu.txt | 4 +- requirements/test/cuda.txt | 2 +- requirements/test/rocm.txt | 2 +- requirements/test/xpu.txt | 2 +- rust/Cargo.lock | 4 +- rust/Cargo.toml | 2 +- .../src/tool/deepseek_dsml/deepseek_v41.rs | 4 +- .../deepseek_v41/structural_tag.rs | 108 ----------------- .../tool/deepseek_dsml/deepseek_v41/tests.rs | 10 +- rust/src/parser/src/tool/hy/structural_tag.rs | 12 +- .../src/unified/kimi_k3/structural_tag.rs | 25 ++-- .../test_structural_tag_registry.py | 71 +++-------- .../deepseekv41_engine_tool_parser.py | 2 +- vllm/tool_parsers/structural_tag_registry.py | 110 +----------------- 15 files changed, 57 insertions(+), 303 deletions(-) delete mode 100644 rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs diff --git a/requirements/common.txt b/requirements/common.txt index ff691ebe47..b55f680b76 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -26,7 +26,7 @@ lm-format-enforcer == 0.11.3 llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x" outlines_core == 0.2.14 lark == 1.2.2 -xgrammar >= 0.2.1, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" +xgrammar == 0.2.7; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" typing_extensions >= 4.10 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317 partial-json-parser # used for parsing partial JSON outputs diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 996caab96a..c1d4913953 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -43,7 +43,7 @@ anyio==4.14.1 # sse-starlette # starlette # watchfiles -apache-tvm-ffi==0.1.9 +apache-tvm-ffi==0.1.14.post0 # via xgrammar arctic-inference==0.1.1 # via -r requirements/test/cuda.in @@ -1298,7 +1298,7 @@ word2number==1.1 # via lm-eval wrapt==1.17.2 # via smart-open -xgrammar==0.2.3 +xgrammar==0.2.7 # via -r requirements/test/../common.txt xxhash==3.5.0 # via diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index ea94558000..3207d395aa 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -1409,7 +1409,7 @@ word2number==1.1 # via lm-eval wrapt==1.17.2 # via smart-open -xgrammar==0.2.3 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 631c3952fd..c961b6294a 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -1360,7 +1360,7 @@ word2number==1.1 # via lm-eval wrapt==2.1.2 # via smart-open -xgrammar==0.2.1 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 4e4fd628b6..d9572389eb 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -1197,7 +1197,7 @@ werkzeug==3.1.8 # via schemathesis word2number==1.1 # via lm-eval -xgrammar==0.2.3 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/rust/Cargo.lock b/rust/Cargo.lock index dee3f30789..b4917a303b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6596,9 +6596,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xgrammar-structural-tag" -version = "0.2.0+xgrammar.0.2.4.dd729e7" +version = "0.3.0+xgrammar.0.2.7.82505d0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d24c842efc3c24e9756aa426d530cbdac0980e49af223cb384e276e981ca0a" +checksum = "3864005575c2ab76057f789898f59fb9fa1378d07c5ceabc24764e012d0e90bb" dependencies = [ "auto_impl", "serde", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dde065e790..ea8b813619 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -153,7 +153,7 @@ vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } vllm-tracing = { path = "src/tracing" } winnow = { version = "1.0.2", features = ["simd"] } -xgrammar-structural-tag = "0.2.0" +xgrammar-structural-tag = "0.3.0" zeromq = { version = "0.6.0", default-features = false, features = [ "tokio-runtime", "all-transport", diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs index 44658b1d44..be68b6d2e5 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs @@ -4,8 +4,6 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput}; -mod structural_tag; - /// Tool parser for DeepSeek V4.1's spaced DSML tags. /// /// Arguments are emitted only after a full `invoke` block is parsed. @@ -27,7 +25,7 @@ impl ToolParser for DeepSeekV41ToolParser { } fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { - Some(&structural_tag::DeepSeekV41StructuralTagBuilder) + Some(xgrammar_structural_tag::Model::DeepSeekV41.builder()) } fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs deleted file mode 100644 index d5fb6118fb..0000000000 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -//! Visible-text grammar for DeepSeek V4.1 tool calls. -//! -//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with -//! V4.1's spaced DSML tags and schema-unconstrained parameter values. - -use serde_json::Value; -use xgrammar_structural_tag::Result; -use xgrammar_structural_tag::builders::{ - StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, -}; -use xgrammar_structural_tag::format::{ - Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat, -}; -use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam}; - -use super::DsmlTokens; - -const TOKENS: DsmlTokens = DsmlTokens::V41; - -pub(super) struct DeepSeekV41StructuralTagBuilder; - -impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder { - fn build(&self, ctx: StructuralTagContext<'_>) -> Result { - let tags = ctx - .function_tools - .iter() - .map(|tool| call_tag(tool, ctx.options)) - .collect::>(); - let excludes = if ctx.options.exclude_special_tokens { - &["", ""][..] - } else { - &[] - }; - let format = match ctx.tool_choice { - BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes), - BuilderToolChoice::Auto => Format::TriggeredTags( - TriggeredTagsFormat::new( - &[TOKENS.tool_calls_start], - vec![TagFormat::new( - format!("{}\n", TOKENS.tool_calls_start), - Format::tags_with_separator(tags, "", true, false), - TOKENS.tool_calls_end, - )], - ) - .with_excludes(excludes), - ), - BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![ - Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)), - Format::tags_with_separator( - tags, - "", - true, - ctx.tool_choice == BuilderToolChoice::Forced, - ), - Format::const_string(TOKENS.tool_calls_end), - ]), - }; - // Reasoning boundaries remain owned by the engine reasoner; serving - // calls this visible-text builder with reasoning=false. - Ok(StructuralTag::new(format)) - } -} - -fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat { - // TODO: enforce tool.parameters and strict schema semantics once V4.1 - // argument conversion is available. This builder intentionally ignores - // parameters and strict: it constrains tool names and DSML/value syntax - // only. Parameter names, presence, uniqueness and value schemas remain - // unconstrained, including for strict=true. Serving still uses strict to - // decide whether auto tool choice activates a grammar. - TagFormat::new( - format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name), - Format::star(parameter(options)), - format!("{}\n", TOKENS.invoke_end), - ) -} - -fn parameter(options: StructuralTagOptions) -> Format { - Format::tag( - format!("{} name=\"", TOKENS.parameter_start), - Format::sequence(vec![ - Format::regex(r#"[^"]+"#), - Format::const_string("\" string=\""), - Format::or(vec![ - Format::sequence(vec![ - Format::const_string("true\">"), - Format::any_text_excluding(&[ - TOKENS.parameter_end, - TOKENS.invoke_end, - TOKENS.tool_calls_end, - ]), - ]), - Format::sequence(vec![ - Format::const_string("false\">"), - Format::JsonSchema( - JsonSchemaFormat::new(Value::Bool(true)) - .with_any_order(options.any_order) - .with_max_whitespace_cnt(options.max_whitespace_cnt), - ), - ]), - ]), - ]), - format!("{}\n", TOKENS.parameter_end), - ) -} diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs index 5a6758f5c9..208d88ea33 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs @@ -102,7 +102,7 @@ fn v4_tags_remain_text_in_v41_dialect() { } #[test] -fn structural_tag_ignores_parameter_schemas_and_strict() { +fn structural_tag_respects_parameter_schemas_and_strict() { let parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); let builder = parser.structural_tag_builder().unwrap(); let options = StructuralTagOptions::default().with_reasoning(false); @@ -128,7 +128,13 @@ fn structural_tag_ignores_parameter_schemas_and_strict() { ] { let mut function = FunctionDefinition::new("lookup").with_parameters(parameters); function.strict = strict; - assert_eq!(build(function), expected); + let tag = build(function); + if strict == Some(false) { + assert_eq!(tag, expected); + } else { + assert_ne!(tag, expected); + } + assert!(tag.to_json_string().unwrap().contains("deepseek_v4_1_xml")); } } } diff --git a/rust/src/parser/src/tool/hy/structural_tag.rs b/rust/src/parser/src/tool/hy/structural_tag.rs index cce98d5b09..e883acbafd 100644 --- a/rust/src/parser/src/tool/hy/structural_tag.rs +++ b/rust/src/parser/src/tool/hy/structural_tag.rs @@ -219,7 +219,7 @@ mod tests { ToolChoice::required(), ); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); } #[test] @@ -238,7 +238,7 @@ mod tests { ToolChoice::required(), ); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"tags_with_separator","tags":[{"begin":"get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","",""]},{"type":"const_string","value":""}]}]},"end":""}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":""}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""}]}]},"end":""}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":""}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); } #[test] @@ -259,7 +259,7 @@ mod tests { ); let value = serde_json::to_value(tag).unwrap(); - expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"limit"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq( + expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"limit"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq( &serde_json::to_string( &value["format"]["elements"][1]["tags"][0]["content"], ) @@ -276,8 +276,8 @@ mod tests { let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap(); let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap(); - expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":[""],"tags":[{"begin":"\n","content":{"type":"tags_with_separator","tags":[{"begin":"search\n","content":{"type":"any_text","excludes":[]},"end":""},{"begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); + expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":[""],"tags":[{"type":"tag","begin":"\n","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"search\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); } #[test] @@ -314,7 +314,7 @@ mod tests { ); let value = serde_json::to_value(tag).unwrap(); - expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"tenant"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq( + expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"tenant"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq( &serde_json::to_string( &value["format"]["elements"][1]["tags"][0]["content"], ) diff --git a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs index 0a86a274e1..fc30f455e9 100644 --- a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs +++ b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs @@ -6,14 +6,14 @@ use serde_json::{Map, Value}; use xgrammar_structural_tag::Result; use xgrammar_structural_tag::builders::{ - StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, + ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, }; use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat}; use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters}; use super::{ ARG_CLOSE, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN, - RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, TOOLS_CLOSE, TOOLS_OPEN, + RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN, }; pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder = @@ -50,14 +50,23 @@ impl StructuralTagBuilder for KimiK3StructuralTagBuilder { } } -fn response_prefix(reasoning: bool) -> Vec { +fn response_prefix(reasoning: ReasoningMode) -> Vec { let mut elements = Vec::new(); - if reasoning { - elements.push(Format::tag( - "", + if reasoning != ReasoningMode::Disabled { + let prefix = Format::tag( + if reasoning == ReasoningMode::Auto { + THINK_OPEN + } else { + "" + }, Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]), THINK_CLOSE, - )); + ); + elements.push(if reasoning == ReasoningMode::Auto { + Format::optional(prefix) + } else { + prefix + }); elements.push(Format::const_string(RESPONSE_OPEN)); } else { elements.push(Format::optional(Format::const_string(RESPONSE_OPEN))); @@ -386,7 +395,7 @@ mod tests { ) .unwrap(); - expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"]},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); } #[test] diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index c216b398cc..3221d1fb2c 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -6,7 +6,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock import pytest -from xgrammar import Grammar, StructuralTag, builtin_structural_tag +from xgrammar import Grammar, StructuralTag from xgrammar.testing import _is_grammar_accept_string from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -23,6 +23,7 @@ from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser from vllm.tool_parsers.deepseekv32_engine_tool_parser import ( DeepSeekV32EngineToolParser, ) +from vllm.tool_parsers.deepseekv41_engine_tool_parser import DeepSeekV41EngineToolParser from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser @@ -83,51 +84,9 @@ def test_supported_structural_tag_models_include_vllm_builtins(): assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS -@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"]) -def test_deepseek_v41_older_xgrammar_keeps_format_constraints( - choice, - sample_tools_strict, - monkeypatch, -): - monkeypatch.setattr( - builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None, raising=False - ) - tool_choice = ( - ChatCompletionNamedToolChoiceParam( - function=ChatCompletionNamedFunction(name=choice) - ) - if choice == "get_weather" - else choice - ) - tag = get_model_structural_tag( - "deepseek_v41", sample_tools_strict, tool_choice, reasoning=False - ) - grammar = Grammar.from_structural_tag(tag) - begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n' - end = "\n" - parameter = ( - '<|DSML| parameter name="undeclared" string="false">' - '{"nested": [true, null, 1.5]}\n' - ) - # Required city is omitted; extra and repeated fields remain unconstrained. - assert _is_grammar_accept_string(grammar, begin + end) - assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end) - assert not _is_grammar_accept_string( - grammar, - begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end, - ) - assert not _is_grammar_accept_string( - grammar, (begin + end).replace('name="get_weather"', 'name="unknown"') - ) - assert not _is_grammar_accept_string( - grammar, begin + parameter.replace("| parameter", "|parameter") + end - ) - assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto") - - def test_deepseek_v41_named_choice_emits_one_call(sample_tools): tag = get_model_structural_tag( - "deepseek_v41", + "deepseek_v4_1", sample_tools, ChatCompletionNamedToolChoiceParam( function=ChatCompletionNamedFunction(name="get_weather") @@ -147,19 +106,14 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools): grammar, "\n\n<|DSML| calls>\n" + call * 2 + "" ) assert ( - get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False) + get_model_structural_tag("deepseek_v4_1", sample_tools, "auto", reasoning=False) is None ) -@pytest.mark.skipif( - not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"), - reason="Requires XGrammar's DeepSeek V4.1 builtin", -) @pytest.mark.parametrize("choice", ["auto", "required", "get_weather"]) -@pytest.mark.parametrize("reasoning", [False, True]) def test_deepseek_v41_constrains_parameters_after_reasoning( - sample_tools_strict, choice, reasoning + sample_tools_strict, choice ): """Strict calls must enforce the schema without requiring another think close.""" tool_choice = ( @@ -171,7 +125,7 @@ def test_deepseek_v41_constrains_parameters_after_reasoning( ) sample_tools_strict[0].function.parameters["additionalProperties"] = False tag = get_model_structural_tag( - "deepseek_v41", sample_tools_strict, tool_choice, reasoning=reasoning + "deepseek_v4_1", sample_tools_strict, tool_choice, reasoning=False ) grammar = Grammar.from_structural_tag(tag) begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n' @@ -195,14 +149,10 @@ def test_deepseek_v41_constrains_parameters_after_reasoning( assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto") -@pytest.mark.skipif( - not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"), - reason="Requires XGrammar's DeepSeek V4.1 builtin", -) def test_deepseek_v41_non_strict_parallel_calls_keep_typed_dsml(sample_tools): sample_tools[0].function.strict = False tag = get_model_structural_tag( - "deepseek_v41", sample_tools, "required", reasoning=False + "deepseek_v4_1", sample_tools, "required", reasoning=False ) grammar = Grammar.from_structural_tag(tag) call = ( @@ -571,6 +521,7 @@ def test_get_model_structural_tag_supports_named_tool_choice( (DeepSeekV31ToolParser, "deepseek_v3_1"), (DeepSeekV32EngineToolParser, "deepseek_v3_2"), (DeepSeekV4EngineToolParser, "deepseek_v4"), + (DeepSeekV41EngineToolParser, "deepseek_v4_1"), (Glm47MoeModelToolParser, "glm_4_7"), (Hermes2ProToolParser, "hermes"), (KimiK2ToolParser, "kimi"), @@ -638,7 +589,11 @@ def test_get_structural_tag_disables_reasoning( assert captured == [False] +@pytest.mark.parametrize( + "parser_cls", [Qwen3EngineToolParser, DeepSeekV41EngineToolParser] +) def test_unified_parser_get_structural_tag_disables_reasoning( + parser_cls, monkeypatch: pytest.MonkeyPatch, sample_tools_strict: list[ChatCompletionToolsParam], ): @@ -654,7 +609,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning( ) class TestParser(DelegatingParser): - tool_parser_cls = Qwen3EngineToolParser + tool_parser_cls = parser_cls request = ChatCompletionRequest( messages=[], diff --git a/vllm/tool_parsers/deepseekv41_engine_tool_parser.py b/vllm/tool_parsers/deepseekv41_engine_tool_parser.py index a4c699aeab..a94883d06a 100644 --- a/vllm/tool_parsers/deepseekv41_engine_tool_parser.py +++ b/vllm/tool_parsers/deepseekv41_engine_tool_parser.py @@ -5,4 +5,4 @@ from vllm.parser.engine.registered_adapters import DeepSeekV41ParserToolAdapter class DeepSeekV41EngineToolParser(DeepSeekV41ParserToolAdapter): # type: ignore[valid-type, misc] - structural_tag_model = "deepseek_v41" + structural_tag_model = "deepseek_v4_1" diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 5d65b186fe..ba86395b7a 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -10,7 +10,6 @@ from openai.types.responses.tool import Tool as ResponsesTool from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed from openai.types.responses.tool_choice_function import ToolChoiceFunction from xgrammar import StructuralTag, normalize_tool_choice -from xgrammar import builtin_structural_tag as xgrammar_builtin_structural_tag from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag from xgrammar.openai_tool_call_schema import ( BuiltinToolParam, @@ -71,11 +70,10 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( "deepseek_v3_2", "glm_4_7", "deepseek_v4", + "deepseek_v4_1", } ) -VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( - {"deepseek_v41", "hermes", "hy_v4", "kimi_k3"} -) +VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"}) SUPPORTED_STRUCTURAL_TAG_MODELS = ( XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS ) @@ -270,110 +268,6 @@ def get_function_parameters(function) -> dict[str, Any] | bool: return function.parameters if function.parameters is not None else True -_V41_CALLS_START = "<|DSML| calls>" -_V41_CALLS_END = "" -_V41_INVOKE_END = "" -_V41_PARAMETER_END = "" - - -@register_vllm_structural_tag("deepseek_v41") -def get_deepseek_v41_structural_tag( - tools: list[FunctionToolParam], - builtin_tools: list[BuiltinToolParam], - tool_choice: SimplifiedToolChoice, - reasoning: bool, - token_suffix: str = "", -) -> StructuralTag: - # Serving enables this visible-text grammar after the reasoning boundary. - builder = getattr( - xgrammar_builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None - ) - if builder is not None: - return builder( - tools=tools, - builtin_tools=builtin_tools, - tool_choice=tool_choice, - reasoning="disabled", - ) - - del builtin_tools, reasoning, token_suffix - - # Compatibility with xgrammar releases without the V4.1 builtin. This constrains - # tool names and DSML/value syntax only. Parameter names, presence, uniqueness - # and value schemas remain unconstrained, including for strict=true. The request - # layer still uses strict to decide whether auto tool choice activates a grammar. - parameter = TagFormat( - begin='<|DSML| parameter name="', - content=SequenceFormat( - elements=[ - RegexFormat(pattern=r'[^"]+'), - ConstStringFormat(value='" string="'), - OrFormat( - elements=[ - SequenceFormat( - elements=[ - ConstStringFormat(value='true">'), - AnyTextFormat( - excludes=[ - _V41_PARAMETER_END, - _V41_INVOKE_END, - _V41_CALLS_END, - ] - ), - ] - ), - SequenceFormat( - elements=[ - ConstStringFormat(value='false">'), - JSONSchemaFormat(json_schema=True), - ] - ), - ] - ), - ] - ), - end=f"{_V41_PARAMETER_END}\n", - ) - calls = TagsWithSeparatorFormat( - tags=[ - TagFormat( - begin=f'<|DSML| invoke name="{tool.function.name}">\n', - content=StarFormat(content=parameter), - end=f"{_V41_INVOKE_END}\n", - ) - for tool in tools - ], - separator="", - at_least_one=True, - stop_after_first=tool_choice == "forced", - ) - if tool_choice == "auto": - return StructuralTag( - format=TriggeredTagsFormat( - triggers=[_V41_CALLS_START], - tags=[ - TagFormat( - begin=f"{_V41_CALLS_START}\n", - content=calls, - end=_V41_CALLS_END, - ) - ], - excludes=["", ""], - ) - if tools - else AnyTextFormat(excludes=["", ""]) - ) - return StructuralTag( - format=SequenceFormat( - elements=[ - ConstStringFormat(value=f"\n\n{_V41_CALLS_START}\n"), - calls, - ConstStringFormat(value=_V41_CALLS_END), - ] - ) - ) - - def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]: arguments_field_prefix = '", "arguments": ' formats = [ From e562003adbb9c4eb177194d1318eb2e6862a8e15 Mon Sep 17 00:00:00 2001 From: Aditya Ganesh Kumar Date: Fri, 18 Sep 2026 19:46:41 +0800 Subject: [PATCH 095/143] [Tokenizer] Drop dead Mistral tokenizer shims for transformers#41962 (#56325) Signed-off-by: Aditya Ganesh Kumar Co-authored-by: Claude Opus 5 Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- vllm/tokenizers/mistral.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/vllm/tokenizers/mistral.py b/vllm/tokenizers/mistral.py index 66b69089f0..4ffefe0db6 100644 --- a/vllm/tokenizers/mistral.py +++ b/vllm/tokenizers/mistral.py @@ -354,22 +354,13 @@ class MistralTokenizer(TokenizerLike): "`text_pair` is not supported by `MistralTokenizer.__call__`." ) - encoded = self.transformers_tokenizer( + return self.transformers_tokenizer( text=text, text_pair=text_pair, add_special_tokens=add_special_tokens, truncation=truncation, max_length=max_length, ) - # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962 - # is in, revert to only call self.transformers_tokenizer(...). - # Hack to fix wrongly added eos token, when fix will be supported the condition - # below will be False even before the revert is done. - if encoded["input_ids"] and encoded["input_ids"][-1] == self.eos_token_id: - encoded["input_ids"].pop(-1) - if attention_mask := encoded.get("attention_mask"): - attention_mask.pop(-1) - return encoded @property def vocab(self) -> list[str]: @@ -389,8 +380,10 @@ class MistralTokenizer(TokenizerLike): max_length: int | None = None, add_special_tokens: bool = True, ) -> list[int]: - # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962 - # is in, directly call self.transformers_tokenizer.encode(...). + # NOTE: transformers guards truncation with `if max_length and ...`, so + # `max_length=0` returns the full sequence instead of an empty one. vLLM + # treats `truncate_prompt_tokens=0` as an empty prompt, so keep slicing + # here until that is fixed upstream. encoded = self.tokenizer.encode(text, bos=add_special_tokens, eos=False) if truncation is not False and max_length is not None: @@ -437,11 +430,6 @@ class MistralTokenizer(TokenizerLike): def decode( self, ids: Sequence[int] | int, skip_special_tokens: bool = False ) -> str: - # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962 - # is in, directly call self.transformers_tokenizer.decode(...). - if isinstance(ids, int): - ids = [ids] - return self.transformers_tokenizer.decode( ids, skip_special_tokens=skip_special_tokens ) From 2b8a5495a88a9a44a9d92e8c4f4aae3b55b6b74b Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Fri, 18 Sep 2026 06:50:01 -0500 Subject: [PATCH 096/143] [Mypy] Fix mypy typing for N/O models (#54142) Signed-off-by: Taneem Ibrahim --- tools/pre_commit/mypy.py | 2 - .../layers/mamba/mamba_utils.py | 10 +- vllm/model_executor/models/aimv2.py | 19 +- vllm/model_executor/models/interfaces.py | 7 + .../model_executor/models/nano_nemotron_vl.py | 112 ++++++++--- vllm/model_executor/models/nemotron_h.py | 9 +- vllm/model_executor/models/nemotron_h_mtp.py | 24 ++- vllm/model_executor/models/nemotron_vl.py | 19 +- vllm/model_executor/models/nvlm_d.py | 1 + vllm/model_executor/models/olmo_hybrid.py | 2 + vllm/model_executor/models/opencua.py | 12 +- vllm/model_executor/models/openpangu.py | 6 +- vllm/model_executor/models/openpangu_mtp.py | 17 +- vllm/model_executor/models/openpangu_vl.py | 175 ++++++++++-------- vllm/model_executor/models/openvla.py | 1 + vllm/model_executor/models/opt.py | 2 + vllm/model_executor/models/ovis.py | 13 +- vllm/model_executor/models/ovis2_5.py | 29 ++- .../processors/nano_nemotron_vl.py | 4 +- 19 files changed, 321 insertions(+), 143 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 234acc0298..71dd929d37 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,8 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[nN]", - r"vllm/model_executor/models/[oO]", r"vllm/model_executor/models/[qQ]", r"vllm/model_executor/models/[uU]", r"vllm/model_executor/models/[vV]", diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index db7e119560..8c89488d2d 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -222,12 +222,18 @@ class MambaStateShapeCalculator: @classmethod def append_replayssm_ring( cls, - base_shapes: tuple[tuple[int, ...], ...], + base_shapes: tuple[tuple[int, int], tuple[int, int, int]], n_groups: int, tp_world_size: int, logical_window: int, backend: MambaBackendEnum, - ) -> tuple[tuple[int, ...], ...]: + ) -> tuple[ + tuple[int, int], + tuple[int, int, int], + tuple[int, int, int], + tuple[int, int], + tuple[int, int, int], + ]: """Append the physical ReplaySSM ring shapes. ``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``; diff --git a/vllm/model_executor/models/aimv2.py b/vllm/model_executor/models/aimv2.py index 3461b57137..4e31b99944 100644 --- a/vllm/model_executor/models/aimv2.py +++ b/vllm/model_executor/models/aimv2.py @@ -26,7 +26,10 @@ from vllm.transformers_utils.configs.ovis import AIMv2Config class AIMv2SwiGLUFFN(nn.Module): def __init__( - self, config: AIMv2Config, quant_config: QuantizationConfig, prefix: str + self, + config: AIMv2Config, + quant_config: QuantizationConfig | None, + prefix: str, ): super().__init__() hidden_features = config.intermediate_size @@ -91,7 +94,10 @@ class AIMv2ViTPreprocessor(nn.Module): class AIMv2Attention(nn.Module): def __init__( - self, config: AIMv2Config, quant_config: QuantizationConfig, prefix: str + self, + config: AIMv2Config, + quant_config: QuantizationConfig | None, + prefix: str, ): super().__init__() self.config = config @@ -144,7 +150,10 @@ class AIMv2Attention(nn.Module): class AIMv2Block(nn.Module): def __init__( - self, config: AIMv2Config, quant_config: QuantizationConfig, prefix: str + self, + config: AIMv2Config, + quant_config: QuantizationConfig | None, + prefix: str, ): super().__init__() self.attn = AIMv2Attention( @@ -166,7 +175,7 @@ class AIMv2Transformer(nn.Module): def __init__( self, config: AIMv2Config, - quant_config: QuantizationConfig, + quant_config: QuantizationConfig | None, *, require_post_norm: bool | None = None, prefix: str = "", @@ -206,7 +215,7 @@ class AIMv2Model(torch.nn.Module): def __init__( self, config: AIMv2Config, - quant_config: QuantizationConfig, + quant_config: QuantizationConfig | None, *, require_post_norm: bool | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 1cefd63dd6..21a09cb7ba 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -91,6 +91,13 @@ MambaStateShapes: TypeAlias = ( tuple[int, int, int], tuple[int, int, int], ] + | tuple[ + tuple[int, int], + tuple[int, int, int], + tuple[int, int, int], + tuple[int, int], + tuple[int, int, int], + ] ) diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 6e500c0c2c..fee431617c 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -12,7 +12,7 @@ import warnings from collections.abc import Iterable, Mapping, Sequence from functools import cached_property from io import BytesIO -from typing import Annotated, Literal, TypeAlias +from typing import Annotated, Literal, TypeAlias, TypedDict, cast import torch import torch.nn as nn @@ -20,7 +20,11 @@ from transformers import BatchFeature, PretrainedConfig from vllm import envs from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import ( + AudioDummyOptions, + BaseDummyOptions, + VideoDummyOptions, +) from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.logger import init_logger from vllm.lora.layers.base import BaseLayerWithLoRA @@ -114,7 +118,7 @@ class NanoNemotronVLAudioFeatureInputs(TensorSchema): type: Literal["audio_features"] = "audio_features" input_audio_features: Annotated[torch.Tensor, TensorShape("c", "t", "f")] feature_attention_mask: Annotated[torch.Tensor, TensorShape("c", "t")] - audio_num_clips: list[int] + audio_num_clips: list[int] | torch.Tensor class NanoNemotronVLImagePixelInputs(TensorSchema): @@ -195,6 +199,12 @@ NanoNemotronVLVideoInputs: TypeAlias = ( ) +class NanoNemotronVLMultiModalInputs(TypedDict, total=False): + images: NanoNemotronVLImageInputs | None + videos: NanoNemotronVLVideoInputs | None + audios: NanoNemotronVLAudioFeatureInputs | None + + class NanoNemotronVLProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor: # Not accepted by NanoNemotronVLProcessor.__init__ @@ -465,8 +475,11 @@ class NanoNemotronVLMultiModalProcessor( if isinstance(images, ImageEmbeddingItems): feature_size = images.get_feature_size(item_idx) elif self.info.is_dynamic_tiler: - feature_size = out_mm_data["num_tokens_per_image"][item_idx] + num_tokens_per_image = out_mm_data["num_tokens_per_image"] + assert isinstance(num_tokens_per_image, list) + feature_size = num_tokens_per_image[item_idx] else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) max_num_tiles = hf_processor.max_num_tiles feature_size = hf_processor.get_num_image_tokens( @@ -530,10 +543,12 @@ class NanoNemotronVLMultiModalProcessor( assert isinstance(num_patches, int) T = hf_processor.video_temporal_patch_size + num_tubelets: int | None if T > 1 and num_patches is not None: num_tubelets = math.ceil(num_patches / T) else: num_tubelets = num_patches + assert num_tubelets is not None video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate if video_pruning_rate is not None and video_pruning_rate > 0.0: @@ -635,6 +650,7 @@ class NanoNemotronVLMultiModalProcessor( audio_items: list[AudioItem] = [] has_audio: list[bool] = [] for idx, metadata in enumerate(metadata_list): + assert metadata is not None video_bytes = metadata.get("original_video_bytes") if video_bytes is None or len(video_bytes) == 0: raise ValueError( @@ -661,21 +677,25 @@ class NanoNemotronVLMultiModalProcessor( # Create a new VideoProcessorItems with metadata that does not contain # the large video bytes, to avoid modifying the input `mm_items`. - new_metadata_list = [ - {k: v for k, v in meta.items() if k != "original_video_bytes"} - for meta in metadata_list - ] + new_metadata_list = metadata_list.copy() + for idx, metadata in enumerate(metadata_list): + assert metadata is not None + new_metadata_list[idx] = { + key: value + for key, value in metadata.items() + if key != "original_video_bytes" + } new_videos = VideoProcessorItems(data=videos.data, metadata=new_metadata_list) - audio_parsed = {} + audio_parsed = MultiModalDataItems() if audio_items: audio_parsed = self.data_parser.parse_mm_data({"audio": audio_items}) # Create a new MultiModalDataItems with the new video and audio items. new_mm_items_dict = {**mm_items, **audio_parsed, "video": new_videos} - mm_items = MultiModalDataItems(new_mm_items_dict) + new_mm_items = MultiModalDataItems(new_mm_items_dict) - return mm_items, audio_items, has_audio + return new_mm_items, audio_items, has_audio def apply( self, @@ -800,7 +820,7 @@ class NanoNemotronVLDummyInputsBuilder( ) videos = [v.copy() for v in videos] - video_items = [] + video_items: list[VideoItem] = [] for video in videos: video_num_frames = video.shape[0] video_metadata = { @@ -869,6 +889,9 @@ class NanoNemotronVLDummyInputsBuilder( target_num_frames = num_frames num_videos = mm_counts.get("video", 0) video_overrides = mm_options.get("video") + assert video_overrides is None or isinstance( + video_overrides, VideoDummyOptions + ) dummy_video = { "video": self._get_dummy_videos( width=video_width, @@ -884,6 +907,9 @@ class NanoNemotronVLDummyInputsBuilder( if sound_config := self.info.sound_config: num_audios = mm_counts.get("audio", 0) audio_overrides = mm_options.get("audio") if mm_options else None + assert audio_overrides is None or isinstance( + audio_overrides, AudioDummyOptions + ) tokens_per_audio = max(1, seq_len // max(num_audios, 1)) max_audio_num_samples = MAX_AUDIO_LEN_S * sound_config.sampling_rate calculated_max_audio_num_samples = ParakeetExtractor.audio_length( @@ -946,6 +972,7 @@ class NemotronH_Nano_VL_V2( model_config = vllm_config.model_config config = model_config.hf_config multimodal_config = model_config.multimodal_config + assert multimodal_config is not None image_size = config.force_image_size patch_size = config.patch_size self.patch_size = patch_size @@ -1138,17 +1165,26 @@ class NemotronH_Nano_VL_V2( return None if self.dynamic_resolution: + assert isinstance(pixel_values_flat, (list, torch.Tensor)) pixel_values_flat = DynamicResolutionImageTiler.stack( pixel_values_flat, self.patch_size ) + imgs_sizes = kwargs.pop("imgs_sizes") + num_tokens_per_image = kwargs.pop("num_tokens_per_image") + assert isinstance(imgs_sizes, list) + assert isinstance(num_tokens_per_image, list) return NanoNemotronVLImagePixelInputsDynamic( - pixel_values_flat=pixel_values_flat, **kwargs + pixel_values_flat=pixel_values_flat, + imgs_sizes=imgs_sizes, + num_tokens_per_image=num_tokens_per_image, ) else: + assert isinstance(pixel_values_flat, torch.Tensor) + image_num_patches = kwargs.pop("image_num_patches") + assert isinstance(image_num_patches, torch.Tensor) return NanoNemotronVLImagePixelInputs( pixel_values_flat=pixel_values_flat, - num_patches=kwargs.pop("image_num_patches"), - **kwargs, + num_patches=image_num_patches, ) def _process_image_input_dynamic( @@ -1166,7 +1202,8 @@ class NemotronH_Nano_VL_V2( return image_embeds.split(num_tokens_per_image) def _process_image_input( - self, image_input: NanoNemotronVLImagePixelInputs + self, + image_input: NanoNemotronVLImagePixelInputs | NanoNemotronVLVideoPixelInputs, ) -> tuple[torch.Tensor, ...]: image_embeds = self.extract_feature(image_input["pixel_values_flat"]) num_patches = image_input["num_patches"] @@ -1362,7 +1399,8 @@ class NemotronH_Nano_VL_V2( # of them than max_num_batched_tokens. # Embed them with the base weights - # the LoRA delta is undefined for tokens with no mapping entry. - embed_tokens = self.get_language_model().model.embed_tokens + language_model = cast(NemotronHForCausalLM, self.get_language_model()) + embed_tokens = language_model.model.embed_tokens if isinstance(embed_tokens, BaseLayerWithLoRA): embed_tokens = embed_tokens.base_layer text_embeddings = embed_tokens(repl_token_ids) @@ -1388,26 +1426,29 @@ class NemotronH_Nano_VL_V2( return None if video_embeds is not None: + assert isinstance(video_embeds, (torch.Tensor, list)) return NanoNemotronVLVideoEmbeddingInputs( type="video_embeds", data=video_embeds, ) if pixel_values_flat_video is not None: - if torch.is_tensor(frames_indices): + if isinstance(frames_indices, torch.Tensor): frames_indices = frames_indices.flatten() else: + assert isinstance(frames_indices, (list, tuple)) frames_indices = torch.cat([f.flatten() for f in frames_indices], dim=0) - if torch.is_tensor(frame_duration_ms): + if isinstance(frame_duration_ms, torch.Tensor): frame_duration_ms = frame_duration_ms.flatten() else: + assert isinstance(frame_duration_ms, (list, tuple)) frame_duration_ms = torch.cat( [f.flatten() for f in frame_duration_ms], dim=0 ) if ( - torch.is_tensor(pixel_values_flat_video) + isinstance(pixel_values_flat_video, torch.Tensor) and pixel_values_flat_video.ndim == 5 ): # batched._reduce_data stacked same-shape videos into @@ -1415,9 +1456,11 @@ class NemotronH_Nano_VL_V2( # same-H,W cat path below handles it uniformly. pixel_values_flat_video = list(pixel_values_flat_video) - if not torch.is_tensor(pixel_values_flat_video): + if not isinstance(pixel_values_flat_video, torch.Tensor): + assert isinstance(pixel_values_flat_video, (list, tuple)) pixel_values_flat_video = torch.cat(pixel_values_flat_video, dim=0) + assert isinstance(video_num_patches, torch.Tensor) expected_h = pixel_values_flat_video.shape[-2] expected_w = pixel_values_flat_video.shape[-1] num_frames = video_num_patches[0].item() @@ -1434,8 +1477,10 @@ class NemotronH_Nano_VL_V2( raise AssertionError("This line should be unreachable.") - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - modalities = {} + def _parse_and_validate_multimodal_inputs( + self, **kwargs: object + ) -> NanoNemotronVLMultiModalInputs: + modalities: NanoNemotronVLMultiModalInputs = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. for input_key in kwargs: @@ -1458,8 +1503,17 @@ class NemotronH_Nano_VL_V2( ) and "audios" not in modalities ): + input_audio_features = kwargs.get("input_audio_features") + feature_attention_mask = kwargs.get("feature_attention_mask") + audio_num_clips = kwargs.get("audio_num_clips") + assert isinstance(input_audio_features, torch.Tensor) + assert isinstance(feature_attention_mask, torch.Tensor) + assert isinstance(audio_num_clips, (list, torch.Tensor)) modalities["audios"] = NanoNemotronVLAudioFeatureInputs( - **kwargs, validate=False + input_audio_features=input_audio_features, + feature_attention_mask=feature_attention_mask, + audio_num_clips=audio_num_clips, + validate=False, ) return modalities @@ -1479,23 +1533,26 @@ class NemotronH_Nano_VL_V2( for modality in modalities: if modality == "images": image_input = modalities["images"] - if image_input["type"] == "image_embeds": + assert image_input is not None + if isinstance(image_input, NanoNemotronVLImageEmbeddingInputs): image_embeddings = image_input["data"] - elif self.dynamic_resolution: - assert image_input["type"] == "pixel_values_dynamic" + elif isinstance(image_input, NanoNemotronVLImagePixelInputsDynamic): image_embeddings = self._process_image_input_dynamic(image_input) else: image_embeddings = self._process_image_input(image_input) multimodal_embeddings += tuple(image_embeddings) if modality == "videos": video_input = modalities["videos"] + assert video_input is not None if video_input["type"] == "video_embeds": video_embeddings = video_input["data"] else: + assert isinstance(video_input, NanoNemotronVLVideoPixelInputs) video_embeddings = self._process_video_input(video_input) multimodal_embeddings += tuple(video_embeddings) if modality == "audios": audio_input = modalities["audios"] + assert audio_input is not None audio_embeddings = self._process_audio_input(audio_input) multimodal_embeddings += tuple(audio_embeddings) @@ -1538,6 +1595,7 @@ class NemotronH_Nano_VL_V2( def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): mm_config = self.model_config.multimodal_config + assert mm_config is not None load_multimodal_weights = not all( mm_config.get_limit_per_prompt(modality) == 0 for modality in ("image", "video", "audio") diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index e71b7d8e6c..e5451d16ba 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -68,6 +68,7 @@ from vllm.model_executor.models.interfaces import ( EagleModelMixin, HasInnerState, IsHybrid, + MambaStateShapes, MixtureOfExperts, SupportsEagle, SupportsEagle3, @@ -138,6 +139,7 @@ class NemotronHMoE(nn.Module): prefix: str = "", ): super().__init__() + assert parallel_config is not None self.tp_size = get_tensor_model_parallel_world_size() self.routed_scaling_factor = config.routed_scaling_factor @@ -188,6 +190,8 @@ class NemotronHMoE(nn.Module): prefix=f"{prefix}.shared_experts", ) + self.fc1_latent_proj: ReplicatedLinear | None + self.fc2_latent_proj: ReplicatedLinear | None if self.use_latent_moe: self.fc1_latent_proj = ReplicatedLinear( input_size=config.hidden_size, @@ -722,7 +726,7 @@ class NemotronHForCausalLM( SupportsReplaySSM, ): # Relevant only if self.has_moe is True - is_non_gated_moe: bool = True + is_non_gated_moe = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={"backbone": "model", "mtp": None}, @@ -773,7 +777,7 @@ class NemotronHForCausalLM( def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[tuple[int, ...], ...]: + ) -> MambaStateShapes: """Calculate shapes for Mamba's convolutional and state caches. Args: @@ -858,6 +862,7 @@ class NemotronHForCausalLM( self.moe_layers.append(layer.mixer.experts) self.num_moe_layers = len(self.moe_layers) + assert example_moe is not None self.num_logical_experts = example_moe.n_logical_experts self.num_physical_experts = example_moe.n_physical_experts self.num_local_physical_experts = example_moe.n_local_physical_experts # noqa: E501 diff --git a/vllm/model_executor/models/nemotron_h_mtp.py b/vllm/model_executor/models/nemotron_h_mtp.py index 96f1b6685e..6b9de63009 100644 --- a/vllm/model_executor/models/nemotron_h_mtp.py +++ b/vllm/model_executor/models/nemotron_h_mtp.py @@ -92,10 +92,12 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer): def forward( self, - inputs_embeds: torch.Tensor, positions: torch.Tensor, hidden_states: torch.Tensor, residual: torch.Tensor | None = None, + *, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, ) -> tuple[torch.Tensor, torch.Tensor | None]: # Start projections (Fusion) if self.has_start_projections: @@ -179,10 +181,11 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer): def forward( self, - inputs_embeds: torch.Tensor, - positions: torch.Tensor, hidden_states: torch.Tensor, residual: torch.Tensor | None = None, + *, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, ) -> tuple[torch.Tensor, torch.Tensor | None]: # Start projections (Fusion) if self.has_start_projections: @@ -305,13 +308,14 @@ class NemotronHMultiTokenPredictor(nn.Module): def forward( self, - input_ids: torch.Tensor, + input_ids: torch.Tensor | None, positions: torch.Tensor, hidden_states: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor | IntermediateTensors: if inputs_embeds is None: + assert input_ids is not None inputs_embeds = self.get_input_embeddings(input_ids) residual = None @@ -386,6 +390,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): @staticmethod def _find_quant_config(*args, **kwargs) -> QuantizationConfig | None: vllm_config = kwargs.get("vllm_config") + assert isinstance(vllm_config, VllmConfig) return get_draft_quant_config(vllm_config) def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: @@ -393,14 +398,15 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): def forward( self, - input_ids: torch.Tensor, + input_ids: torch.Tensor | None, positions: torch.Tensor, - hidden_states: torch.Tensor, + hidden_states: torch.Tensor | None = None, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, **kwargs: object, ) -> torch.Tensor: """Forward - applies attention-based MTP.""" + assert hidden_states is not None hidden_states = self.model( input_ids, positions, @@ -473,9 +479,10 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): name = name.replace("backbone.", "model.") if "scale" in name or "zero_point" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name # Handle stacked parameters (qkv_proj) for attention layers is_stacked = False @@ -547,6 +554,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): param = params_dict[name] weight_loader = getattr(param, "weight_loader", default_weight_loader) + assert weight_loader is not None weight_loader(param, loaded_weight) loaded_params.add(name) diff --git a/vllm/model_executor/models/nemotron_vl.py b/vllm/model_executor/models/nemotron_vl.py index bf05848858..7b2777a710 100644 --- a/vllm/model_executor/models/nemotron_vl.py +++ b/vllm/model_executor/models/nemotron_vl.py @@ -138,7 +138,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor prefix=maybe_prefix(prefix, "language_model"), ) - self.img_context_token_id = None + self.img_context_token_id: int | None = None self.visual_token_mask = None self.make_empty_intermediate_tensors = ( @@ -146,7 +146,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor ) def _patch_quant_config( - self, config: PretrainedConfig, quant_config: QuantizationConfig + self, config: PretrainedConfig, quant_config: QuantizationConfig | None ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back @@ -518,12 +518,21 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling) use_head=False, ) - def _init_mlp1(self, config: PretrainedConfig) -> nn.Module: + def _init_mlp1( + self, + config: PretrainedConfig, + vit_hidden_size: int | None = None, + vision_projection_hidden_size: int | None = None, + ) -> nn.Module: """Override to use different MLP structure for embedding model.""" + if vit_hidden_size is None: + vit_hidden_size = config.vision_config.hidden_size + if vision_projection_hidden_size is None: + vision_projection_hidden_size = config.get_text_config().hidden_size return super()._init_mlp1( config, - vit_hidden_size=config.vision_config.hidden_size, - vision_projection_hidden_size=config.get_text_config().hidden_size, + vit_hidden_size=vit_hidden_size, + vision_projection_hidden_size=vision_projection_hidden_size, ) def _call_vision_model(self, pixel_values: torch.Tensor) -> torch.Tensor: diff --git a/vllm/model_executor/models/nvlm_d.py b/vllm/model_executor/models/nvlm_d.py index d937fade1d..a69f024a01 100644 --- a/vllm/model_executor/models/nvlm_d.py +++ b/vllm/model_executor/models/nvlm_d.py @@ -128,6 +128,7 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo if isinstance(images, ImageEmbeddingItems): feature_size = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) feature_size = self.info.get_num_image_tokens( image_width=image_size.width, diff --git a/vllm/model_executor/models/olmo_hybrid.py b/vllm/model_executor/models/olmo_hybrid.py index 9a24df5ec9..e32087ea21 100644 --- a/vllm/model_executor/models/olmo_hybrid.py +++ b/vllm/model_executor/models/olmo_hybrid.py @@ -143,6 +143,7 @@ class OlmoHybridAttention(nn.Module): rope_parameters["rope_theta"] is not None ) + self.rotary_emb: nn.Module | None if self._use_rope: self.rotary_emb = get_rope( self.head_dim, @@ -183,6 +184,7 @@ class OlmoHybridAttention(nn.Module): q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) q, k = self._apply_qk_norm(q, k) if self._use_rope: + assert self.rotary_emb is not None q, k = self.rotary_emb(positions, q, k) attn_output = self.attn(q, k, v) output, _ = self.o_proj(attn_output) diff --git a/vllm/model_executor/models/opencua.py b/vllm/model_executor/models/opencua.py index 6a875c813c..6b12d1d3fb 100644 --- a/vllm/model_executor/models/opencua.py +++ b/vllm/model_executor/models/opencua.py @@ -7,6 +7,7 @@ """Inference-only OpenCUA-7B model compatible with HuggingFace weights.""" +import typing from collections.abc import Mapping, Sequence from typing import Any @@ -27,6 +28,7 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, PromptReplacement, PromptUpdate, @@ -178,7 +180,13 @@ class OpenCUAMultiModalProcessor(BaseMultiModalProcessor[OpenCUAProcessingInfo]) ] -class OpenCUADummyInputsBuilder(Qwen2VLDummyInputsBuilder): +if typing.TYPE_CHECKING: + _OpenCUADummyInputsBuilderBase = BaseDummyInputsBuilder[OpenCUAProcessingInfo] +else: + _OpenCUADummyInputsBuilderBase = Qwen2VLDummyInputsBuilder + + +class OpenCUADummyInputsBuilder(_OpenCUADummyInputsBuilderBase): def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: num_images = mm_counts.get("image", 0) @@ -220,7 +228,7 @@ class OpenCUAForConditionalGeneration(Qwen2_5_VLForConditionalGeneration): nn.Module.__init__(self) config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.config = config diff --git a/vllm/model_executor/models/openpangu.py b/vllm/model_executor/models/openpangu.py index b8937f2e09..4aa603b458 100644 --- a/vllm/model_executor/models/openpangu.py +++ b/vllm/model_executor/models/openpangu.py @@ -175,6 +175,7 @@ class OpenPanguMoE(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size + self.shared_experts: OpenPanguMLP | None if config.n_shared_experts is not None: intermediate_size = config.moe_intermediate_size * config.n_shared_experts self.shared_experts = OpenPanguMLP( @@ -283,7 +284,7 @@ class OpenPanguMLAAttention(nn.Module): ) self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps) self.q_b_proj = ColumnParallelLinear( - q_lora_rank, + self.q_lora_rank, self.num_heads * self.qk_head_dim, bias=False, quant_config=quant_config, @@ -568,6 +569,9 @@ class OpenPanguSinkAttention(nn.Module): self.qk_nope_dim = getattr(config, "qk_nope_dim", None) self.qk_rope_dim = getattr(config, "qk_rope_dim", None) self.v_channels = getattr(config, "v_channels", None) + assert self.qk_nope_dim is not None + assert self.qk_rope_dim is not None + assert self.v_channels is not None self.head_dim = self.qk_rope_dim + self.qk_nope_dim self.q_size = self.num_heads * self.head_dim self.k_size = self.num_kv_heads * self.head_dim diff --git a/vllm/model_executor/models/openpangu_mtp.py b/vllm/model_executor/models/openpangu_mtp.py index d25efd8c03..c91453342a 100644 --- a/vllm/model_executor/models/openpangu_mtp.py +++ b/vllm/model_executor/models/openpangu_mtp.py @@ -52,7 +52,9 @@ class OpenPanguMultiTokenPredictorLayer(DeepSeekMultiTokenPredictorLayer): def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: nn.Module.__init__(self) - config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + config = speculative_config.draft_model_config.hf_config self.config = config quant_config = vllm_config.quant_config @@ -200,10 +202,15 @@ class OpenPanguMTP(nn.Module): break else: for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: + ( + expert_param_name, + expert_weight_name, + expert_id, + expert_shard_id, + ) = mapping + if expert_weight_name not in name: continue - name = name.replace(weight_name, param_name) + name = name.replace(expert_weight_name, expert_param_name) param = params_dict[name] weight_loader = param.weight_loader @@ -211,7 +218,7 @@ class OpenPanguMTP(nn.Module): param, loaded_weight, name, - shard_id=shard_id, + shard_id=expert_shard_id, expert_id=expert_id, ) break diff --git a/vllm/model_executor/models/openpangu_vl.py b/vllm/model_executor/models/openpangu_vl.py index 8b85d0371b..741ec2155f 100644 --- a/vllm/model_executor/models/openpangu_vl.py +++ b/vllm/model_executor/models/openpangu_vl.py @@ -22,7 +22,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from functools import lru_cache, partial -from typing import Annotated, Literal, Optional +from typing import Annotated, Literal, Optional, TypeAlias import torch import torch.nn as nn @@ -627,7 +627,7 @@ class OpenPanguVisionTransformer(nn.Module): pad[-(dim + 1) * 2 + 1] = pad_len return F.pad(w, pad, mode="constant", value=0) - stacked_params_mapping = [ + stacked_params_mapping: list[tuple[str, str, str | int]] = [ ("attn.qkv.", "attn.q.", "q"), ("attn.qkv.", "attn.k.", "k"), ("attn.qkv.", "attn.v.", "v"), @@ -677,15 +677,11 @@ class OpenPanguVLProcessingInfo(Qwen2_5_VLProcessingInfo): def get_hf_config(self): return self.ctx.model_config.hf_config - def get_hf_processor( - self, - *, - min_pixels: int | None = None, - max_pixels: int | None = None, - size: dict[str, int] | None = None, - fps: float | list[float] | None = None, - **kwargs: object, - ): + def get_hf_processor(self, **kwargs: object): + kwargs.pop("min_pixels", None) + kwargs.pop("max_pixels", None) + kwargs.pop("size", None) + fps = kwargs.pop("fps", None) if fps is not None: kwargs["fps"] = fps @@ -747,11 +743,19 @@ class OpenPanguVLVideoEmbeddingInputs(TensorSchema): ] +OpenPanguVLImageInputs: TypeAlias = ( + OpenPanguVLImagePixelInputs | OpenPanguVLImageEmbeddingInputs +) +OpenPanguVLVideoInputs: TypeAlias = ( + OpenPanguVLVideoPixelInputs | OpenPanguVLVideoEmbeddingInputs +) + + class OpenPanguVLMultiModalProcessor(Qwen2_5_VLMultiModalProcessor): def _get_prompt_updates( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, any], + hf_processor_mm_kwargs: Mapping[str, object], out_mm_kwargs: MultiModalKwargsItems, ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) @@ -878,10 +882,12 @@ class OpenPanguVLForConditionalGeneration( self.do_rescale = image_processor.do_rescale self.rescale_factor = image_processor.rescale_factor self.do_normalize = image_processor.do_normalize - self.image_mean = tuple(image_processor.image_mean) - self.image_std = tuple(image_processor.image_std) + self.image_mean: tuple[float, ...] = tuple(image_processor.image_mean) + self.image_std: tuple[float, ...] = tuple(image_processor.image_std) - def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig): + def _maybe_ignore_quant_config( + self, quant_config: QuantizationConfig | None + ) -> QuantizationConfig | None: if isinstance(quant_config, AutoGPTQConfig): return None return quant_config @@ -904,7 +910,9 @@ class OpenPanguVLForConditionalGeneration( else: return torch.concat(mm_input) - def _parse_and_validate_image_input(self, **kwargs: object): + def _parse_and_validate_image_input( + self, **kwargs: object + ) -> OpenPanguVLImageInputs | None: pixel_values = kwargs.pop("pixel_values", None) image_embeds = kwargs.pop("image_embeds", None) image_grid_thw = kwargs.pop("image_grid_thw", None) @@ -932,26 +940,27 @@ class OpenPanguVLForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: - image_embeds = self._validate_and_reshape_mm_tensor( - image_embeds, "image embeds" - ) - image_grid_thw = self._validate_and_reshape_mm_tensor( - image_grid_thw, "image grid_thw" - ) + assert image_embeds is not None + image_embeds = self._validate_and_reshape_mm_tensor( + image_embeds, "image embeds" + ) + image_grid_thw = self._validate_and_reshape_mm_tensor( + image_grid_thw, "image grid_thw" + ) - if not isinstance(image_embeds, torch.Tensor): - raise ValueError( - "Incorrect type of image embeddings. " - f"Got type: {type(image_embeds)}" - ) - return OpenPanguVLImageEmbeddingInputs( - type="image_embeds", - image_embeds=image_embeds, - image_grid_thw=image_grid_thw, + if not isinstance(image_embeds, torch.Tensor): + raise ValueError( + f"Incorrect type of image embeddings. Got type: {type(image_embeds)}" ) + return OpenPanguVLImageEmbeddingInputs( + type="image_embeds", + image_embeds=image_embeds, + image_grid_thw=image_grid_thw, + ) - def _parse_and_validate_video_input(self, **kwargs: object): + def _parse_and_validate_video_input( + self, **kwargs: object + ) -> OpenPanguVLVideoInputs | None: pixel_values_videos = kwargs.pop("pixel_values_videos", None) video_embeds = kwargs.pop("video_embeds", None) video_grid_thw = kwargs.pop("video_grid_thw", None) @@ -973,27 +982,30 @@ class OpenPanguVLForConditionalGeneration( video_grid_thw=video_grid_thw, ) - if video_embeds is not None: - video_embeds = self._validate_and_reshape_mm_tensor( - video_embeds, "video embeds" - ) - video_grid_thw = self._validate_and_reshape_mm_tensor( - video_grid_thw, "video grid_thw" - ) + assert video_embeds is not None + video_embeds = self._validate_and_reshape_mm_tensor( + video_embeds, "video embeds" + ) + video_grid_thw = self._validate_and_reshape_mm_tensor( + video_grid_thw, "video grid_thw" + ) - if not isinstance(video_embeds, torch.Tensor): - raise ValueError( - "Incorrect type of video embeddings. " - f"Got type: {type(video_embeds)}" - ) - return OpenPanguVLVideoEmbeddingInputs( - type="video_embeds", - video_embeds=video_embeds, - video_grid_thw=video_grid_thw, + if not isinstance(video_embeds, torch.Tensor): + raise ValueError( + f"Incorrect type of video embeddings. Got type: {type(video_embeds)}" ) + return OpenPanguVLVideoEmbeddingInputs( + type="video_embeds", + video_embeds=video_embeds, + video_grid_thw=video_grid_thw, + ) - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + def _parse_and_validate_multimodal_inputs( + self, **kwargs: object + ) -> dict[str, OpenPanguVLImageInputs | OpenPanguVLVideoInputs | None]: + mm_input_by_modality: dict[ + str, OpenPanguVLImageInputs | OpenPanguVLVideoInputs | None + ] = {} for input_key in kwargs: if ( input_key in ("pixel_values", "image_embeds") @@ -1019,16 +1031,19 @@ class OpenPanguVLForConditionalGeneration( multimodal_embeddings: tuple[torch.Tensor, ...] = () for modality in mm_input_by_modality: - multimodal_input = mm_input_by_modality[modality] if modality == "image": - vision_embeddings = self._process_image_input(multimodal_input) + image_input = mm_input_by_modality["image"] + assert isinstance(image_input, OpenPanguVLImageInputs) + vision_embeddings = self._process_image_input(image_input) multimodal_embeddings = ( multimodal_embeddings if not vision_embeddings else (multimodal_embeddings + vision_embeddings) ) if modality == "video": - video_embeddings = self._process_video_input(multimodal_input) + video_input = mm_input_by_modality["video"] + assert isinstance(video_input, OpenPanguVLVideoInputs) + video_embeddings = self._process_video_input(video_input) multimodal_embeddings = ( multimodal_embeddings if not video_embeddings @@ -1041,20 +1056,19 @@ class OpenPanguVLForConditionalGeneration( input_ids: torch.Tensor, multimodal_embeddings: MultiModalEmbeddings | None = None, ) -> torch.Tensor: - if multimodal_embeddings is None: - return self.language_model.embed_input_ids(input_ids) + inputs_embeds = self.language_model.embed_input_ids(input_ids) + if multimodal_embeddings is not None: + inputs_embeds = self.embed_input_ids( # type: ignore[call-overload] + input_ids, + inputs_embeds, + multimodal_embeddings, + [self.config.image_token_id, self.config.video_token_id], + ) + return inputs_embeds - mm_token_ids = input_ids.new_tensor( - [self.config.image_token_id, self.config.video_token_id] - ) - is_multimodal = torch.isin(input_ids, mm_token_ids) - return self.embed_input_ids( - input_ids, - multimodal_embeddings, - is_multimodal=is_multimodal, - ) - - def _process_image_input(self, image_input) -> tuple[torch.Tensor, ...]: + def _process_image_input( + self, image_input: OpenPanguVLImageInputs + ) -> tuple[torch.Tensor, ...]: grid_thw = image_input["image_grid_thw"] if grid_thw.ndim != 2: raise ValueError(f"grid_thw.ndim must be 2, but it is {grid_thw.ndim}") @@ -1085,7 +1099,9 @@ class OpenPanguVLForConditionalGeneration( sizes = grid_thw.prod(-1) // merge_size // merge_size return image_embeds.split(sizes.tolist()) - def _process_video_input(self, video_input) -> torch.Tensor: + def _process_video_input( + self, video_input: OpenPanguVLVideoInputs + ) -> tuple[torch.Tensor, ...]: grid_thw = video_input["video_grid_thw"] if grid_thw.ndim != 2: raise ValueError(f"grid_thw.ndim must be 2, but it is {grid_thw.ndim}") @@ -1158,8 +1174,13 @@ class OpenPanguVLForConditionalGeneration( for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset modality = mm_feature.modality + feature_data = mm_feature.data + assert feature_data is not None if modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + grid_thw_item = feature_data["image_grid_thw"] + grid_thw = grid_thw_item.data + assert isinstance(grid_thw, torch.Tensor) + t, h, w = grid_thw.tolist() assert t == 1, f"Image must have 1 frame, got {t}" yield ( modality, @@ -1169,7 +1190,10 @@ class OpenPanguVLForConditionalGeneration( w // spatial_merge_size, ) elif modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + grid_thw_item = feature_data["video_grid_thw"] + grid_thw = grid_thw_item.data + assert isinstance(grid_thw, torch.Tensor) + t, h, w = grid_thw.tolist() yield ( modality, offset, @@ -1281,14 +1305,17 @@ def normalize(image, mean, std): @lru_cache(maxsize=10) def _fuse_mean_std_and_rescale_factor( do_normalize: bool | None = None, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, + image_mean: float | tuple[float, ...] | None = None, + image_std: float | tuple[float, ...] | None = None, do_rescale: bool | None = None, rescale_factor: float | None = None, device: Optional["torch.device"] = None, ) -> tuple: if do_rescale and do_normalize: # Fused rescale and normalize + assert rescale_factor is not None + assert image_mean is not None + assert image_std is not None image_mean = torch.tensor(image_mean, device=device) * (1.0 / rescale_factor) image_std = torch.tensor(image_std, device=device) * (1.0 / rescale_factor) do_rescale = False @@ -1300,8 +1327,8 @@ def rescale_and_normalize( do_rescale: bool, rescale_factor: float, do_normalize: bool, - image_mean: float | list[float], - image_std: float | list[float], + image_mean: float | tuple[float, ...], + image_std: float | tuple[float, ...], dtype: torch.dtype = torch.bfloat16, ) -> "torch.Tensor": """Rescale and normalize images.""" diff --git a/vllm/model_executor/models/openvla.py b/vllm/model_executor/models/openvla.py index 77099ffb6e..add7eac41f 100644 --- a/vllm/model_executor/models/openvla.py +++ b/vllm/model_executor/models/openvla.py @@ -349,6 +349,7 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo]) if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) num_image_tokens = self.info.get_num_image_tokens( image_width=image_size.width, diff --git a/vllm/model_executor/models/opt.py b/vllm/model_executor/models/opt.py index 1efe9eb8a8..4726c72b41 100644 --- a/vllm/model_executor/models/opt.py +++ b/vllm/model_executor/models/opt.py @@ -218,6 +218,7 @@ class OPTDecoder(nn.Module): ) # Project out & in will be replicated if they exist. + self.project_out: ReplicatedLinear | None if config.word_embed_proj_dim != config.hidden_size: self.project_out = ReplicatedLinear( config.hidden_size, @@ -229,6 +230,7 @@ class OPTDecoder(nn.Module): else: self.project_out = None + self.project_in: ReplicatedLinear | None if config.word_embed_proj_dim != config.hidden_size: self.project_in = ReplicatedLinear( config.word_embed_proj_dim, diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index 5563b9a9ce..bd00686b41 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -59,7 +59,12 @@ from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.ovis import OvisProcessor from vllm.utils.tensor_schema import TensorSchema, TensorShape -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP +from .interfaces import ( + MultiModalEmbeddings, + SupportsMultiModal, + SupportsPP, + supports_pp, +) # Cannot find the following number from hf config. IMAGE_TOKEN = "" @@ -281,7 +286,7 @@ class OvisProcessingInfo(BaseProcessingInfo): # minus 1 for presented image token return (patch_grid_length // hidden_stride) ** 2 - 1 - def get_image_pad_token(self) -> str: + def get_image_pad_token(self) -> str | None: hf_text_config = self.get_hf_config().get_text_config() text_model_type = hf_text_config.model_type return IMAGE_PAD_TOKEN_MAP.get(text_model_type) @@ -439,8 +444,10 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP): text_model_type = self.config.get_text_config().model_type self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type] + language_model = self.get_language_model() + assert supports_pp(language_model) self.make_empty_intermediate_tensors = ( - self.get_language_model().make_empty_intermediate_tensors + language_model.make_empty_intermediate_tensors ) def _parse_and_validate_image_input( diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 0b8a2c5f84..588c06b8de 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -4,14 +4,14 @@ from collections.abc import Iterable, Mapping from functools import partial -from typing import Annotated, Literal +from typing import Annotated, Literal, TypedDict import torch import torch.nn as nn from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig @@ -40,7 +40,12 @@ from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor from vllm.utils.tensor_schema import TensorSchema, TensorShape -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP +from .interfaces import ( + MultiModalEmbeddings, + SupportsMultiModal, + SupportsPP, + supports_pp, +) IMAGE_TOKEN = "" VIDEO_TOKEN = "